ruihan/qwen38-flash-next-rtx6000d
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Document MTP 2 speed and memory tradeoff; retain stable default

Browse Source
This commit is contained in:
ruihan
2026-09-18 11:00:52 +08:00
parent 7e53821920
commit 80f70ec6f8
14 changed files with 758 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/runtime/resources-mtp.jsonl
+282
View File
@@ -0,0 +1,282 @@
{"time": 1789699784.8557913, "available_gib": 173.268310546875, "oom_kill": 0, "gpu": "80181, 85651, 0, 31", "stop": false}
{"time": 1789699786.8766131, "available_gib": 174.90249252319336, "oom_kill": 0, "gpu": "80181, 85651, 0, 31", "stop": false}
{"time": 1789699788.8985825, "available_gib": 175.68295288085938, "oom_kill": 0, "gpu": "80181, 85651, 0, 31", "stop": false}
{"time": 1789699790.91918, "available_gib": 175.66968536376953, "oom_kill": 0, "gpu": "80181, 85651, 0, 31", "stop": false}
{"time": 1789699792.9395878, "available_gib": 175.68845748901367, "oom_kill": 0, "gpu": "80181, 85651, 0, 31", "stop": false}
{"time": 1789699800.8114684, "available_gib": 179.87840270996094, "oom_kill": 0, "gpu": "3, 85651, 0, 31", "stop": false}
{"time": 1789699802.834334, "available_gib": 242.46122360229492, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false}
{"time": 1789699804.8571448, "available_gib": 242.3736572265625, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false}
{"time": 1789699806.8817687, "available_gib": 242.27896881103516, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false}
{"time": 1789699808.9074936, "available_gib": 242.22193145751953, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789699810.932665, "available_gib": 242.20484161376953, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789699812.957609, "available_gib": 242.17076110839844, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789699814.9819207, "available_gib": 242.11393356323242, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789699817.005921, "available_gib": 242.04299545288086, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789699819.0289435, "available_gib": 241.47203063964844, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false}
{"time": 1789699821.062485, "available_gib": 241.26908493041992, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false}
{"time": 1789699823.0869603, "available_gib": 241.22373580932617, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789699825.1192734, "available_gib": 240.7252311706543, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789699827.1438193, "available_gib": 241.52180099487305, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789699829.1859672, "available_gib": 241.16375732421875, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789699831.2292492, "available_gib": 240.9925537109375, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789699833.2707527, "available_gib": 240.5682601928711, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789699835.3125954, "available_gib": 240.0120506286621, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789699837.3552454, "available_gib": 239.8315200805664, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789699839.3975637, "available_gib": 239.30066299438477, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789699841.4175756, "available_gib": 239.12970352172852, "oom_kill": 0, "gpu": "703, 85651, 0, 29", "stop": false}
{"time": 1789699843.437184, "available_gib": 238.37808227539062, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789699845.4552722, "available_gib": 233.91911697387695, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789699847.4740174, "available_gib": 229.94686126708984, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789699849.4928708, "available_gib": 226.03619384765625, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789699851.5111096, "available_gib": 222.09746932983398, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789699853.5298238, "available_gib": 218.20297622680664, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789699855.5481977, "available_gib": 214.18994140625, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699857.5675807, "available_gib": 210.1709442138672, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699859.5862136, "available_gib": 206.13785552978516, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699861.6048002, "available_gib": 202.08191680908203, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699863.6235912, "available_gib": 198.0450782775879, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699865.6427789, "available_gib": 193.9893913269043, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699867.6612194, "available_gib": 189.95485305786133, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699869.679868, "available_gib": 185.93933868408203, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699871.6985507, "available_gib": 181.9157257080078, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699873.7171915, "available_gib": 177.8902702331543, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699875.736203, "available_gib": 174.32772064208984, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789699877.7578192, "available_gib": 173.71036529541016, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789699879.7790232, "available_gib": 173.81313705444336, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789699881.8009658, "available_gib": 173.8381690979004, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789699883.822374, "available_gib": 173.9555320739746, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789699885.8436148, "available_gib": 173.97101211547852, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699887.8657157, "available_gib": 173.9713020324707, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699889.8866572, "available_gib": 173.96550369262695, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699891.906882, "available_gib": 173.96137237548828, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699893.927697, "available_gib": 173.96295547485352, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699895.9530587, "available_gib": 173.97043991088867, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699897.973925, "available_gib": 173.98381805419922, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699899.9951556, "available_gib": 173.99149703979492, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699902.016985, "available_gib": 174.02708435058594, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699904.037938, "available_gib": 174.0359764099121, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699906.0589933, "available_gib": 174.04009628295898, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699908.0798283, "available_gib": 174.04407119750977, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699910.100448, "available_gib": 174.0757827758789, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699912.121421, "available_gib": 174.0846405029297, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699914.1430233, "available_gib": 174.09698104858398, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699916.1640205, "available_gib": 174.0961456298828, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699918.1853118, "available_gib": 174.11655044555664, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789699920.2060194, "available_gib": 174.06558990478516, "oom_kill": 0, "gpu": "78809, 85651, 0, 32", "stop": false}
{"time": 1789699922.226115, "available_gib": 173.94388961791992, "oom_kill": 0, "gpu": "82531, 85651, 0, 31", "stop": false}
{"time": 1789699924.2462852, "available_gib": 173.94985961914062, "oom_kill": 0, "gpu": "82559, 85651, 0, 31", "stop": false}
{"time": 1789699926.2655146, "available_gib": 173.94927978515625, "oom_kill": 0, "gpu": "82559, 85651, 0, 31", "stop": false}
{"time": 1789699928.2854214, "available_gib": 173.38904571533203, "oom_kill": 0, "gpu": "82613, 85651, 0, 31", "stop": false}
{"time": 1789699930.3055093, "available_gib": 173.25407791137695, "oom_kill": 0, "gpu": "79131, 85651, 0, 31", "stop": false}
{"time": 1789699932.3254104, "available_gib": 172.94384002685547, "oom_kill": 0, "gpu": "79131, 85651, 0, 31", "stop": false}
{"time": 1789699934.3455124, "available_gib": 172.37591171264648, "oom_kill": 0, "gpu": "81095, 85651, 0, 31", "stop": false}
{"time": 1789699936.3656113, "available_gib": 172.36243057250977, "oom_kill": 0, "gpu": "81095, 85651, 0, 31", "stop": false}
{"time": 1789699938.3857331, "available_gib": 172.38327407836914, "oom_kill": 0, "gpu": "81095, 85651, 0, 31", "stop": false}
{"time": 1789699940.40693, "available_gib": 172.39754104614258, "oom_kill": 0, "gpu": "82317, 85651, 100, 40", "stop": false}
{"time": 1789699942.4280798, "available_gib": 172.43234252929688, "oom_kill": 0, "gpu": "82317, 85651, 100, 42", "stop": false}
{"time": 1789699944.4486258, "available_gib": 172.45121002197266, "oom_kill": 0, "gpu": "82317, 85651, 100, 43", "stop": false}
{"time": 1789699946.468317, "available_gib": 172.4884033203125, "oom_kill": 0, "gpu": "82319, 85651, 0, 34", "stop": false}
{"time": 1789699948.4880044, "available_gib": 172.42709350585938, "oom_kill": 0, "gpu": "82319, 85651, 0, 34", "stop": false}
{"time": 1789699950.507294, "available_gib": 172.47681045532227, "oom_kill": 0, "gpu": "82319, 85651, 0, 33", "stop": false}
{"time": 1789699952.5260417, "available_gib": 172.41959762573242, "oom_kill": 0, "gpu": "82319, 85651, 0, 33", "stop": false}
{"time": 1789699954.5465488, "available_gib": 172.17332077026367, "oom_kill": 0, "gpu": "80519, 85651, 0, 33", "stop": false}
{"time": 1789699956.567162, "available_gib": 171.95952224731445, "oom_kill": 0, "gpu": "80519, 85651, 0, 33", "stop": false}
{"time": 1789699958.5875938, "available_gib": 171.57903671264648, "oom_kill": 0, "gpu": "80553, 85651, 0, 33", "stop": false}
{"time": 1789699960.6085618, "available_gib": 171.84549713134766, "oom_kill": 0, "gpu": "80189, 85651, 0, 33", "stop": false}
{"time": 1789699962.6290948, "available_gib": 172.0185546875, "oom_kill": 0, "gpu": "80239, 85651, 0, 33", "stop": false}
{"time": 1789699964.6503246, "available_gib": 172.32759857177734, "oom_kill": 0, "gpu": "80197, 85651, 0, 32", "stop": false}
{"time": 1789699966.676846, "available_gib": 172.48663330078125, "oom_kill": 0, "gpu": "80197, 85651, 0, 32", "stop": false}
{"time": 1789699968.697056, "available_gib": 174.07500839233398, "oom_kill": 0, "gpu": "80197, 85651, 0, 32", "stop": false}
{"time": 1789699970.7173376, "available_gib": 174.9847412109375, "oom_kill": 0, "gpu": "80197, 85651, 0, 32", "stop": false}
{"time": 1789699972.737336, "available_gib": 175.0500602722168, "oom_kill": 0, "gpu": "80197, 85651, 0, 32", "stop": false}
{"time": 1789699974.7615445, "available_gib": 175.09698104858398, "oom_kill": 0, "gpu": "80197, 85651, 0, 32", "stop": false}
{"time": 1789699982.4932203, "available_gib": 180.97797012329102, "oom_kill": 0, "gpu": "3, 85651, 0, 32", "stop": false}
{"time": 1789699984.5346766, "available_gib": 242.31150436401367, "oom_kill": 0, "gpu": "3, 85651, 0, 31", "stop": false}
{"time": 1789699986.5568547, "available_gib": 242.04119110107422, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false}
{"time": 1789699988.5815885, "available_gib": 242.06488800048828, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false}
{"time": 1789699990.6061108, "available_gib": 242.0366439819336, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false}
{"time": 1789699992.630566, "available_gib": 241.96244049072266, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false}
{"time": 1789699994.6744077, "available_gib": 241.48827743530273, "oom_kill": 0, "gpu": "3, 85651, 0, 30", "stop": false}
{"time": 1789699996.7181695, "available_gib": 241.28020095825195, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false}
{"time": 1789699998.7595315, "available_gib": 241.0977783203125, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false}
{"time": 1789700000.801106, "available_gib": 240.78686904907227, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false}
{"time": 1789700002.8429427, "available_gib": 240.23579788208008, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false}
{"time": 1789700004.8777978, "available_gib": 240.06826400756836, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false}
{"time": 1789700006.927845, "available_gib": 239.48531341552734, "oom_kill": 0, "gpu": "3, 85651, 0, 29", "stop": false}
{"time": 1789700008.9487274, "available_gib": 238.55954360961914, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700010.9686577, "available_gib": 234.62310791015625, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700012.9886026, "available_gib": 230.06606674194336, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700015.0087287, "available_gib": 226.08770751953125, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700017.028833, "available_gib": 222.08605575561523, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700019.0487628, "available_gib": 218.09417724609375, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700021.068506, "available_gib": 214.07280731201172, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700023.0881393, "available_gib": 210.02310180664062, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700025.1080492, "available_gib": 206.0057144165039, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700027.1281407, "available_gib": 201.98720932006836, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700029.1478603, "available_gib": 197.94964599609375, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700031.1675708, "available_gib": 193.9510269165039, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700033.1873636, "available_gib": 189.92237854003906, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700035.207234, "available_gib": 185.88830947875977, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700037.2265067, "available_gib": 181.86507415771484, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700039.2458131, "available_gib": 177.82470703125, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700041.2655752, "available_gib": 174.4015007019043, "oom_kill": 0, "gpu": "4285, 85651, 0, 31", "stop": false}
{"time": 1789700043.287092, "available_gib": 174.46878051757812, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700045.3077006, "available_gib": 174.47719192504883, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700047.328204, "available_gib": 174.48582077026367, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700049.3485699, "available_gib": 174.45703887939453, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700051.3691437, "available_gib": 174.4439811706543, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700053.3893752, "available_gib": 174.42322158813477, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700055.4098642, "available_gib": 174.4094581604004, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700057.4303467, "available_gib": 174.40129852294922, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700059.4506557, "available_gib": 176.9820098876953, "oom_kill": 0, "gpu": "76995, 85651, 0, 31", "stop": false}
{"time": 1789700061.4705217, "available_gib": 177.71730422973633, "oom_kill": 0, "gpu": "76995, 85651, 0, 31", "stop": false}
{"time": 1789700063.490828, "available_gib": 177.7105712890625, "oom_kill": 0, "gpu": "76995, 85651, 0, 31", "stop": false}
{"time": 1789700065.5109022, "available_gib": 177.6934700012207, "oom_kill": 0, "gpu": "76995, 85651, 0, 31", "stop": false}
{"time": 1789700067.53109, "available_gib": 177.72324752807617, "oom_kill": 0, "gpu": "76995, 85651, 0, 31", "stop": false}
{"time": 1789700074.1739979, "available_gib": 194.71649169921875, "oom_kill": 0, "gpu": "0, 85651, 0, 31", "stop": false}
{"time": 1789700076.1959038, "available_gib": 242.32111358642578, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false}
{"time": 1789700078.2183936, "available_gib": 242.2687110900879, "oom_kill": 0, "gpu": "0, 85651, 0, 30", "stop": false}
{"time": 1789700080.2433918, "available_gib": 242.21891403198242, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789700082.2683237, "available_gib": 242.2096939086914, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789700084.2929466, "available_gib": 242.16974639892578, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789700086.3170736, "available_gib": 242.17188262939453, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789700088.3417797, "available_gib": 242.07259368896484, "oom_kill": 0, "gpu": "0, 85651, 0, 29", "stop": false}
{"time": 1789700090.3671076, "available_gib": 241.59612274169922, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false}
{"time": 1789700092.4088504, "available_gib": 241.66643524169922, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789700094.4781263, "available_gib": 241.26079177856445, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789700096.5219333, "available_gib": 241.02852249145508, "oom_kill": 0, "gpu": "3, 85651, 0, 28", "stop": false}
{"time": 1789700098.5478, "available_gib": 242.32376098632812, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false}
{"time": 1789700100.5917299, "available_gib": 242.18223571777344, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false}
{"time": 1789700102.6330614, "available_gib": 242.18046188354492, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false}
{"time": 1789700104.6753898, "available_gib": 242.11222076416016, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false}
{"time": 1789700106.7176976, "available_gib": 242.0906639099121, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false}
{"time": 1789700108.7608912, "available_gib": 242.08220672607422, "oom_kill": 0, "gpu": "0, 85651, 0, 28", "stop": false}
{"time": 1789700110.804005, "available_gib": 242.03167343139648, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false}
{"time": 1789700112.8491402, "available_gib": 241.9895896911621, "oom_kill": 0, "gpu": "0, 85651, 0, 27", "stop": false}
{"time": 1789700114.8809855, "available_gib": 241.31512069702148, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false}
{"time": 1789700116.9228864, "available_gib": 241.4677734375, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false}
{"time": 1789700118.9670393, "available_gib": 241.2608985900879, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false}
{"time": 1789700121.008349, "available_gib": 240.94713592529297, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false}
{"time": 1789700123.0505216, "available_gib": 240.44035720825195, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false}
{"time": 1789700125.0926425, "available_gib": 240.10882949829102, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false}
{"time": 1789700127.135921, "available_gib": 239.90147399902344, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false}
{"time": 1789700129.1780832, "available_gib": 239.06969833374023, "oom_kill": 0, "gpu": "3, 85651, 0, 27", "stop": false}
{"time": 1789700131.1975424, "available_gib": 239.08580017089844, "oom_kill": 0, "gpu": "2915, 85651, 0, 28", "stop": false}
{"time": 1789700133.216903, "available_gib": 236.34324645996094, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700135.2349987, "available_gib": 232.00969314575195, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700137.2533488, "available_gib": 228.20048904418945, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700139.2720008, "available_gib": 224.26771926879883, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700141.2904346, "available_gib": 220.33390426635742, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700143.3088236, "available_gib": 216.4178237915039, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700145.3273628, "available_gib": 212.44153594970703, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700147.3463542, "available_gib": 208.3773651123047, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700149.3647766, "available_gib": 204.3053741455078, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700151.3832946, "available_gib": 200.27174758911133, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700153.4018042, "available_gib": 196.2129783630371, "oom_kill": 0, "gpu": "4285, 85651, 0, 29", "stop": false}
{"time": 1789700155.4205408, "available_gib": 192.17596054077148, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700157.4392505, "available_gib": 188.12572860717773, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700159.4579098, "available_gib": 184.0816307067871, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700161.4766073, "available_gib": 180.06254959106445, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700163.4951246, "available_gib": 176.02914428710938, "oom_kill": 0, "gpu": "4285, 85651, 0, 30", "stop": false}
{"time": 1789700165.5196788, "available_gib": 173.59284591674805, "oom_kill": 0, "gpu": "4413, 85651, 32, 30", "stop": false}
{"time": 1789700167.5409636, "available_gib": 174.47128295898438, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700169.5620625, "available_gib": 174.4740219116211, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700171.5827205, "available_gib": 174.47975540161133, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700173.6033459, "available_gib": 174.46636581420898, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700175.625155, "available_gib": 174.47333908081055, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700177.6462028, "available_gib": 174.48141479492188, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700179.6672688, "available_gib": 174.48285675048828, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700181.6880727, "available_gib": 174.4776496887207, "oom_kill": 0, "gpu": "76995, 85651, 6, 30", "stop": false}
{"time": 1789700183.7094233, "available_gib": 174.46028518676758, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700185.730227, "available_gib": 174.43238830566406, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700187.7518756, "available_gib": 174.4477767944336, "oom_kill": 0, "gpu": "76995, 85651, 4, 30", "stop": false}
{"time": 1789700189.772606, "available_gib": 174.4392433166504, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700191.793387, "available_gib": 174.43968200683594, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700193.8145614, "available_gib": 174.4402618408203, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700195.8353708, "available_gib": 174.45610427856445, "oom_kill": 0, "gpu": "76995, 85651, 6, 31", "stop": false}
{"time": 1789700197.856705, "available_gib": 174.46376419067383, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700199.8782845, "available_gib": 174.4733009338379, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700201.8992653, "available_gib": 174.47993850708008, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700203.9204733, "available_gib": 174.4811134338379, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700205.9425795, "available_gib": 174.46759414672852, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700207.9636607, "available_gib": 174.4632797241211, "oom_kill": 0, "gpu": "76995, 85651, 4, 31", "stop": false}
{"time": 1789700209.9835656, "available_gib": 174.40138244628906, "oom_kill": 0, "gpu": "78809, 85651, 0, 31", "stop": false}
{"time": 1789700212.0033963, "available_gib": 174.2780303955078, "oom_kill": 0, "gpu": "82531, 85651, 0, 31", "stop": false}
{"time": 1789700214.0232139, "available_gib": 174.26416015625, "oom_kill": 0, "gpu": "82559, 85651, 0, 31", "stop": false}
{"time": 1789700216.0458455, "available_gib": 174.2457504272461, "oom_kill": 0, "gpu": "82587, 85651, 96, 36", "stop": false}
{"time": 1789700218.0656052, "available_gib": 174.06429290771484, "oom_kill": 0, "gpu": "79131, 85651, 0, 31", "stop": false}
{"time": 1789700220.087584, "available_gib": 173.7898826599121, "oom_kill": 0, "gpu": "79131, 85651, 0, 31", "stop": false}
{"time": 1789700222.10688, "available_gib": 173.32328414916992, "oom_kill": 0, "gpu": "81095, 85651, 0, 31", "stop": false}
{"time": 1789700224.126727, "available_gib": 173.26789474487305, "oom_kill": 0, "gpu": "81095, 85651, 0, 31", "stop": false}
{"time": 1789700226.1464965, "available_gib": 173.2430763244629, "oom_kill": 0, "gpu": "81095, 85651, 0, 31", "stop": false}
{"time": 1789700228.1671848, "available_gib": 173.30466079711914, "oom_kill": 0, "gpu": "82317, 85651, 100, 40", "stop": false}
{"time": 1789700230.188617, "available_gib": 173.31119918823242, "oom_kill": 0, "gpu": "82317, 85651, 100, 41", "stop": false}
{"time": 1789700232.208453, "available_gib": 173.28583526611328, "oom_kill": 0, "gpu": "82317, 85651, 100, 42", "stop": false}
{"time": 1789700234.2289078, "available_gib": 173.2372932434082, "oom_kill": 0, "gpu": "82319, 85651, 0, 34", "stop": false}
{"time": 1789700236.2490773, "available_gib": 173.2351951599121, "oom_kill": 0, "gpu": "82319, 85651, 0, 33", "stop": false}
{"time": 1789700238.269133, "available_gib": 173.20354461669922, "oom_kill": 0, "gpu": "82319, 85651, 0, 33", "stop": false}
{"time": 1789700240.2894697, "available_gib": 173.2050018310547, "oom_kill": 0, "gpu": "82319, 85651, 0, 33", "stop": false}
{"time": 1789700242.3098028, "available_gib": 172.9349822998047, "oom_kill": 0, "gpu": "80553, 85651, 0, 33", "stop": false}
{"time": 1789700244.3320792, "available_gib": 173.1237335205078, "oom_kill": 0, "gpu": "80225, 85651, 0, 33", "stop": false}
{"time": 1789700246.352452, "available_gib": 173.32941055297852, "oom_kill": 0, "gpu": "80303, 85651, 0, 33", "stop": false}
{"time": 1789700248.3720224, "available_gib": 173.30435943603516, "oom_kill": 0, "gpu": "83033, 85651, 0, 32", "stop": false}
{"time": 1789700250.3918135, "available_gib": 173.2729606628418, "oom_kill": 0, "gpu": "83033, 85651, 0, 32", "stop": false}
{"time": 1789700252.4116886, "available_gib": 173.2642707824707, "oom_kill": 0, "gpu": "83033, 85651, 0, 32", "stop": false}
{"time": 1789700254.4326909, "available_gib": 173.27268981933594, "oom_kill": 0, "gpu": "83033, 85651, 0, 32", "stop": false}
{"time": 1789700256.4531732, "available_gib": 173.28126907348633, "oom_kill": 0, "gpu": "83035, 85651, 0, 32", "stop": false}
{"time": 1789700258.4731524, "available_gib": 173.13495254516602, "oom_kill": 0, "gpu": "83041, 85651, 0, 32", "stop": false}
{"time": 1789700260.493189, "available_gib": 173.013671875, "oom_kill": 0, "gpu": "83041, 85651, 0, 32", "stop": false}
{"time": 1789700262.5140247, "available_gib": 173.01695251464844, "oom_kill": 0, "gpu": "83087, 85651, 0, 32", "stop": false}
{"time": 1789700264.5341256, "available_gib": 173.04697036743164, "oom_kill": 0, "gpu": "83175, 85651, 0, 32", "stop": false}
{"time": 1789700266.553813, "available_gib": 173.0420799255371, "oom_kill": 0, "gpu": "83175, 85651, 0, 32", "stop": false}
{"time": 1789700268.573798, "available_gib": 173.0691909790039, "oom_kill": 0, "gpu": "83175, 85651, 0, 32", "stop": false}
{"time": 1789700270.59406, "available_gib": 173.10009765625, "oom_kill": 0, "gpu": "83175, 85651, 0, 32", "stop": false}
{"time": 1789700272.6137307, "available_gib": 173.07026290893555, "oom_kill": 0, "gpu": "83175, 85651, 0, 32", "stop": false}
{"time": 1789700274.639571, "available_gib": 173.02909469604492, "oom_kill": 0, "gpu": "83183, 85651, 100, 38", "stop": false}
{"time": 1789700276.65941, "available_gib": 172.89321899414062, "oom_kill": 0, "gpu": "83185, 85651, 0, 33", "stop": false}
{"time": 1789700278.6844745, "available_gib": 172.90272903442383, "oom_kill": 0, "gpu": "83547, 85651, 100, 44", "stop": false}
{"time": 1789700280.710171, "available_gib": 172.9180564880371, "oom_kill": 0, "gpu": "83547, 85651, 100, 44", "stop": false}
{"time": 1789700282.7349055, "available_gib": 172.88683700561523, "oom_kill": 0, "gpu": "83547, 85651, 100, 45", "stop": false}
{"time": 1789700284.7583632, "available_gib": 172.87765884399414, "oom_kill": 0, "gpu": "83547, 85651, 100, 46", "stop": false}
{"time": 1789700286.7827125, "available_gib": 172.8911247253418, "oom_kill": 0, "gpu": "83547, 85651, 100, 47", "stop": false}
{"time": 1789700288.812623, "available_gib": 172.87197875976562, "oom_kill": 0, "gpu": "83719, 85651, 100, 47", "stop": false}
{"time": 1789700290.837241, "available_gib": 172.86443328857422, "oom_kill": 0, "gpu": "84169, 85651, 100, 49", "stop": false}
{"time": 1789700292.857805, "available_gib": 172.86686325073242, "oom_kill": 0, "gpu": "84895, 85651, 47, 42", "stop": false}
{"time": 1789700294.8838098, "available_gib": 172.8665657043457, "oom_kill": 0, "gpu": "84895, 85651, 100, 45", "stop": false}
{"time": 1789700296.9096725, "available_gib": 172.86813735961914, "oom_kill": 0, "gpu": "84895, 85651, 100, 45", "stop": false}
{"time": 1789700298.9358406, "available_gib": 172.8501434326172, "oom_kill": 0, "gpu": "84895, 85651, 100, 45", "stop": false}
{"time": 1789700300.9623625, "available_gib": 172.901123046875, "oom_kill": 0, "gpu": "84895, 85651, 100, 45", "stop": false}
{"time": 1789700302.982952, "available_gib": 172.91755294799805, "oom_kill": 0, "gpu": "84895, 85651, 0, 40", "stop": false}
{"time": 1789700305.0032096, "available_gib": 172.92973709106445, "oom_kill": 0, "gpu": "84895, 85651, 0, 40", "stop": false}
{"time": 1789700307.023144, "available_gib": 172.9438362121582, "oom_kill": 0, "gpu": "84895, 85651, 0, 39", "stop": false}
{"time": 1789700309.043444, "available_gib": 172.9451789855957, "oom_kill": 0, "gpu": "84895, 85651, 0, 38", "stop": false}
{"time": 1789700311.0636864, "available_gib": 172.94405364990234, "oom_kill": 0, "gpu": "84895, 85651, 0, 38", "stop": false}
{"time": 1789700313.0836325, "available_gib": 172.91239166259766, "oom_kill": 0, "gpu": "84895, 85651, 0, 38", "stop": false}
{"time": 1789700315.103711, "available_gib": 172.9315185546875, "oom_kill": 0, "gpu": "84895, 85651, 0, 37", "stop": false}
{"time": 1789700317.12361, "available_gib": 172.93312454223633, "oom_kill": 0, "gpu": "84895, 85651, 0, 37", "stop": false}
{"time": 1789700319.14374, "available_gib": 172.9223403930664, "oom_kill": 0, "gpu": "84895, 85651, 0, 37", "stop": false}
{"time": 1789700321.1637733, "available_gib": 172.91282272338867, "oom_kill": 0, "gpu": "84895, 85651, 0, 36", "stop": false}
{"time": 1789700323.1840858, "available_gib": 172.92692947387695, "oom_kill": 0, "gpu": "84895, 85651, 0, 36", "stop": false}
{"time": 1789700325.2040703, "available_gib": 172.92637252807617, "oom_kill": 0, "gpu": "84895, 85651, 0, 36", "stop": false}
{"time": 1789700327.2239447, "available_gib": 172.9365463256836, "oom_kill": 0, "gpu": "84895, 85651, 0, 36", "stop": false}
{"time": 1789700329.243812, "available_gib": 172.96002960205078, "oom_kill": 0, "gpu": "84895, 85651, 0, 35", "stop": false}
{"time": 1789700331.264149, "available_gib": 172.95891189575195, "oom_kill": 0, "gpu": "84895, 85651, 0, 35", "stop": false}
{"time": 1789700333.2835627, "available_gib": 172.94894790649414, "oom_kill": 0, "gpu": "84895, 85651, 0, 35", "stop": false}
{"time": 1789700335.303892, "available_gib": 172.96074295043945, "oom_kill": 0, "gpu": "84895, 85651, 0, 35", "stop": false}
{"time": 1789700337.3236537, "available_gib": 172.9765968322754, "oom_kill": 0, "gpu": "84895, 85651, 0, 35", "stop": false}
{"time": 1789700339.3438094, "available_gib": 172.9586296081543, "oom_kill": 0, "gpu": "84895, 85651, 0, 34", "stop": false}
{"time": 1789700341.3641162, "available_gib": 172.97516250610352, "oom_kill": 0, "gpu": "84895, 85651, 0, 34", "stop": false}
{"time": 1789700343.3842447, "available_gib": 172.96298599243164, "oom_kill": 0, "gpu": "84895, 85651, 0, 34", "stop": false}
{"time": 1789700345.4077187, "available_gib": 172.9586181640625, "oom_kill": 0, "gpu": "84895, 85651, 0, 34", "stop": false}
{"time": 1789700347.4277773, "available_gib": 172.9544677734375, "oom_kill": 0, "gpu": "84895, 85651, 0, 34", "stop": false}
{"time": 1789700349.4475327, "available_gib": 172.94096755981445, "oom_kill": 0, "gpu": "84895, 85651, 0, 34", "stop": false}
{"time": 1789700351.467543, "available_gib": 172.94974517822266, "oom_kill": 0, "gpu": "84895, 85651, 0, 33", "stop": false}
{"time": 1789700353.487416, "available_gib": 172.9431915283203, "oom_kill": 0, "gpu": "84895, 85651, 0, 33", "stop": false}
{"time": 1789700355.507813, "available_gib": 172.96563339233398, "oom_kill": 0, "gpu": "84895, 85651, 0, 33", "stop": false}
{"time": 1789700357.5280263, "available_gib": 172.95469665527344, "oom_kill": 0, "gpu": "84895, 85651, 0, 33", "stop": false}
{"time": 1789700359.548705, "available_gib": 174.7944564819336, "oom_kill": 0, "gpu": "84895, 85651, 0, 33", "stop": false}
{"time": 1789700361.5688958, "available_gib": 175.5979118347168, "oom_kill": 0, "gpu": "84895, 85651, 0, 33", "stop": false}
{"time": 1789700363.5890248, "available_gib": 175.59072494506836, "oom_kill": 0, "gpu": "84895, 85651, 0, 33", "stop": false}
{"time": 1789700365.6091769, "available_gib": 175.58855438232422, "oom_kill": 0, "gpu": "84895, 85651, 0, 33", "stop": false}
{"time": 1789700367.6296282, "available_gib": 175.56648635864258, "oom_kill": 0, "gpu": "84895, 85651, 0, 33", "stop": false}
{"time": 1789700374.2080383, "available_gib": 193.37168884277344, "oom_kill": 0, "gpu": "3, 85651, 0, 32", "stop": false}