| { |
| "original_ppl": 7.836853981018066, |
| "reduced_ppl": [ |
| { |
| "bit": 1, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 100028.296875 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 83601.1640625 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 92061.140625 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 73126.3125 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 88826.78125 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 88911.1953125 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 101415.1328125 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 112060.359375 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 99527.3984375 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 112942.484375 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 45329.7890625 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 103545.53125 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 75738.6953125 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 129706.734375 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 92175.9609375 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 93578.515625 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 124439.2734375 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 112074.7890625 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 75031.5625 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 62521.05859375 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 74879.6640625 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 104447.65625 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 87225.9296875 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 86935.265625 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 94477.2890625 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 137997.0625 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 85168.9453125 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 106327.9609375 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 111321.1328125 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 93278.4296875 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 109165.90625 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 60157.2265625 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 112575.2578125 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 63213.53125 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 93929.5390625 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 114455.203125 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 102396.4921875 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 114803.4921875 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 106479.578125 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 99373.9453125 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 104597.9765625 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 132235.328125 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 98028.671875 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 123878.734375 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 52708.54296875 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 160839.015625 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 119890.265625 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 142591.71875 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 54418.26953125 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 110809.1171875 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 112228.265625 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 144154.25 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 139704.3125 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 138363.53125 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 104079.5546875 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 73313.3125 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 96951.3046875 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 99314.6328125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 86732.0546875 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 69419.7265625 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 78901.5546875 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 100445.65625 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 96061.5078125 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 90961.0859375 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 91467.8046875 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 142583.0 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 129425.375 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 150609.171875 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 76623.3671875 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 109846.7109375 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 102881.8046875 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 114216.078125 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 102605.484375 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 84844.3515625 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 132192.453125 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 87694.3515625 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 88012.4765625 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 88757.1796875 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 83935.09375 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 82636.4765625 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 131119.703125 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 135234.796875 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 85947.2734375 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 139499.9375 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 120826.1953125 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 103897.96875 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 138357.59375 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 108792.1796875 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 110489.28125 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 67052.59375 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 112562.2734375 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 110496.234375 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 76167.9453125 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 140275.96875 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 92528.5234375 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 121607.4296875 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 70721.2265625 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 80244.796875 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 93308.234375 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 96368.90625 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 78115.1796875 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 75864.1953125 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 94399.03125 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 123373.421875 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 112834.1875 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 98729.703125 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 89207.8671875 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 116006.96875 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 127953.7578125 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 90116.734375 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 95834.578125 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 77265.65625 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 99592.3515625 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 96327.921875 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 104385.4140625 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 111820.796875 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 117226.765625 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 66718.859375 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 92781.234375 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 95134.5078125 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 91213.78125 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 130175.515625 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 94601.625 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 95593.421875 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 92860.4609375 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 100756.3203125 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 102315.859375 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 100177.3203125 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 94189.234375 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 92713.1328125 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 119000.3828125 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 94653.5078125 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 95588.8671875 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 100924.609375 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 102325.421875 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 110875.71875 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 89535.2421875 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 104271.0 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 109607.703125 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 116153.875 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 103546.421875 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 103352.0625 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 104164.4609375 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 104534.25 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 116952.1796875 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 81080.890625 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 111908.0625 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 105461.484375 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 104852.2578125 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 100538.625 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 92973.625 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 99675.390625 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 96183.25 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 100381.8828125 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 105332.9296875 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 107246.1484375 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 111774.2109375 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 107385.75 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 101985.6171875 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 98287.2265625 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 101858.2890625 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 102961.8984375 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 104293.0703125 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 108741.25 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 97056.859375 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 105336.4453125 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 93973.5390625 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 95834.03125 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 103708.1015625 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 106327.15625 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 92967.6796875 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 117210.109375 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 121469.3828125 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 98738.3671875 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 103587.109375 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 102603.234375 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 104255.4921875 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 122396.390625 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 107528.8125 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 105061.75 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 102482.4609375 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 105272.9765625 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 105902.125 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 106416.4296875 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 104918.875 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 111305.640625 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 102500.8359375 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 100910.359375 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 99575.4375 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 100629.3671875 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 99075.578125 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 106700.5625 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 92074.921875 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 102103.1796875 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 97920.3828125 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 107986.6328125 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 103662.4140625 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 105780.1875 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 102874.9375 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 103913.4296875 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 101132.2421875 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 101583.75 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 102753.15625 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 102017.5234375 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 103262.015625 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 103129.75 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 97783.9453125 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 106424.953125 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 105678.7578125 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 98854.640625 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 103172.4453125 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 102745.4140625 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 107614.578125 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 109414.8046875 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 88412.921875 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 93250.3203125 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 103805.0703125 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 107196.25 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 104741.3203125 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 102185.1015625 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 117909.1328125 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 104212.1484375 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 119181.8828125 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 116339.125 |
| } |
| ] |
| }, |
| { |
| "bit": 2, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 228.3678741455078 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 228.46743774414062 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 222.2572021484375 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 211.26962280273438 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 215.52572631835938 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 219.65652465820312 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 209.06570434570312 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 223.21820068359375 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 221.11788940429688 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 213.39540100097656 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 209.99407958984375 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 218.8164520263672 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 219.75196838378906 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 221.50559997558594 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 188.50621032714844 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 187.1911163330078 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 233.6358184814453 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 226.49337768554688 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 214.4426727294922 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 214.77731323242188 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 215.86720275878906 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 217.47842407226562 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 225.78201293945312 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 221.66905212402344 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 217.09828186035156 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 215.488525390625 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 227.7002410888672 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 211.12149047851562 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 222.66986083984375 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 221.27883911132812 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 235.73121643066406 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 219.1390838623047 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 209.91036987304688 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 208.96144104003906 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 223.18743896484375 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 219.43960571289062 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 226.047119140625 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 222.2498016357422 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 222.46185302734375 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 215.72518920898438 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 208.7921142578125 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 212.58201599121094 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 222.00534057617188 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 224.95497131347656 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 216.35894775390625 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 218.5930633544922 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 216.84913635253906 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 221.52291870117188 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 216.25375366210938 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 217.30572509765625 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 223.11370849609375 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 216.8362274169922 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 226.0778350830078 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 221.59730529785156 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 214.74014282226562 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 222.6441650390625 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 222.92185974121094 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 223.34532165527344 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 223.4870147705078 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 219.4259033203125 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 219.3214111328125 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 229.59971618652344 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 221.233154296875 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 227.65899658203125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 224.48294067382812 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 223.63145446777344 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 213.0618133544922 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 221.92742919921875 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 221.87855529785156 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 218.923095703125 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 229.1191864013672 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 225.0966033935547 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 227.00900268554688 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 216.30572509765625 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 221.0341796875 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 220.5524139404297 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 226.80699157714844 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 217.80690002441406 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 224.61444091796875 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 222.60679626464844 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 216.32872009277344 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 218.53457641601562 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 218.1864013671875 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 222.09829711914062 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 225.72451782226562 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 225.82896423339844 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 218.0876922607422 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 209.11355590820312 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 221.56707763671875 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 213.20074462890625 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 219.24945068359375 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 223.7906036376953 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 223.98544311523438 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 219.54417419433594 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 219.72315979003906 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 218.4221649169922 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 225.51979064941406 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 230.84043884277344 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 217.7310791015625 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 220.02069091796875 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 220.4967803955078 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 218.07510375976562 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 217.10997009277344 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 216.0346221923828 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 222.48519897460938 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 221.45236206054688 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 221.23158264160156 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 231.26304626464844 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 224.46058654785156 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 224.06471252441406 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 224.08885192871094 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 222.9420623779297 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 223.28207397460938 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 225.77210998535156 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 218.97447204589844 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 220.190185546875 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 221.8339080810547 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 223.5560760498047 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 225.9592742919922 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 223.25970458984375 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 225.0081787109375 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 225.51344299316406 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 226.3449249267578 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 223.1551971435547 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 222.52784729003906 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 221.00045776367188 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 225.70602416992188 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 225.899169921875 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 225.85479736328125 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 226.9906005859375 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 226.323974609375 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 224.31121826171875 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 227.20437622070312 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 226.37547302246094 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 225.5098876953125 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 228.6000518798828 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 226.0824737548828 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 226.59654235839844 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 225.6342315673828 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 223.24417114257812 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 225.87008666992188 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 225.52528381347656 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 224.75511169433594 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 226.6101531982422 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 223.4763641357422 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 227.87294006347656 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 225.22415161132812 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 226.0972442626953 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 225.25572204589844 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 226.89451599121094 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 224.33132934570312 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 226.04518127441406 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 226.2178192138672 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 226.20896911621094 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 226.49888610839844 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 226.83848571777344 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 227.25509643554688 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 227.216064453125 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 227.22634887695312 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 226.32937622070312 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 225.7090301513672 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 225.5762481689453 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 226.71055603027344 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 225.86599731445312 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 225.4437713623047 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 226.06124877929688 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 225.97427368164062 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 225.8191680908203 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 225.64617919921875 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 225.4522705078125 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 226.14308166503906 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 227.13287353515625 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 225.8455352783203 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 225.1112060546875 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 225.62767028808594 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 225.1337432861328 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 224.3606414794922 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 224.99853515625 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 224.58328247070312 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 225.21051025390625 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 225.2963409423828 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 225.1029510498047 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 227.73324584960938 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 227.08338928222656 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 226.2667999267578 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 225.4050750732422 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 225.28195190429688 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 225.0098876953125 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 225.7315216064453 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 223.64349365234375 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 225.79006958007812 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 227.4889373779297 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 227.6394500732422 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 228.1409454345703 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 229.07460021972656 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 226.2749786376953 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 225.8909912109375 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 226.37998962402344 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 226.64256286621094 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 225.7344207763672 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 225.33372497558594 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 225.9529266357422 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 227.65052795410156 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 228.2270050048828 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 227.05838012695312 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 226.4374237060547 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 224.9395294189453 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 226.60313415527344 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 229.72567749023438 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 226.14036560058594 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 225.7740478515625 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 226.18544006347656 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 227.56976318359375 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 227.54080200195312 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 223.43821716308594 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 226.6251678466797 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 224.72274780273438 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 224.64474487304688 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 226.2241668701172 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 225.1304168701172 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 226.35560607910156 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 229.4119415283203 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 233.0724334716797 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 222.42938232421875 |
| } |
| ] |
| }, |
| { |
| "bit": 3, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 13.056154251098633 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 13.054343223571777 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 13.077469825744629 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 13.02145004272461 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 13.039437294006348 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 13.01038932800293 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 12.998372077941895 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 13.039815902709961 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 13.037182807922363 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 13.07637882232666 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 13.04530143737793 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 13.008739471435547 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 13.04500961303711 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 13.011906623840332 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 12.994876861572266 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 12.986613273620605 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 13.01194953918457 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 13.030412673950195 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 13.009800910949707 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 13.036300659179688 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 12.98751163482666 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 13.033370018005371 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 13.044224739074707 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 13.043557167053223 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 13.003244400024414 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 13.011345863342285 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 13.079253196716309 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 13.028091430664062 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 13.058588981628418 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 13.051467895507812 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 13.022072792053223 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 13.00686264038086 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 12.993618965148926 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 13.017637252807617 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 13.031915664672852 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 13.042054176330566 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 13.038243293762207 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 13.079460144042969 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 13.074601173400879 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 13.008724212646484 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 13.00227165222168 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 13.04793643951416 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 13.020977973937988 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 13.046019554138184 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 13.048234939575195 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 13.06552791595459 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 12.98932933807373 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 13.05797290802002 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 13.01521110534668 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 13.025771141052246 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 13.060395240783691 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 13.020121574401855 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 13.044026374816895 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 13.012030601501465 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 12.986647605895996 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 13.008950233459473 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 13.072696685791016 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 13.080307960510254 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 13.080721855163574 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 13.017451286315918 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 13.000194549560547 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 13.049578666687012 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 13.001651763916016 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 13.059977531433105 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 13.026103973388672 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 13.0299711227417 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 13.05820369720459 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 13.023719787597656 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 13.056319236755371 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 13.070926666259766 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 13.03841781616211 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 13.06108570098877 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 13.035017013549805 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 13.08551025390625 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 13.042689323425293 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 13.03673267364502 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 13.024617195129395 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 13.04966926574707 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 13.044231414794922 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 13.021648406982422 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 13.010563850402832 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 13.015127182006836 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 13.090855598449707 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 13.024585723876953 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 13.044548988342285 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 13.041081428527832 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 13.012505531311035 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 13.02734661102295 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 12.987951278686523 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 13.04334831237793 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 13.009236335754395 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 13.046123504638672 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 13.039216041564941 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 13.055307388305664 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 12.991779327392578 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 13.042813301086426 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 13.034221649169922 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 13.03722095489502 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 13.006742477416992 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 12.986577033996582 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 12.999540328979492 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 13.008174896240234 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 13.015645980834961 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 12.988312721252441 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 12.993650436401367 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 13.044897079467773 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 13.054044723510742 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 13.078455924987793 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 13.01488208770752 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 13.013466835021973 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 13.058675765991211 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 13.0155029296875 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 13.04500675201416 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 13.058393478393555 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 13.05136775970459 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 13.07390022277832 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 12.98832893371582 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 13.051324844360352 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 13.004103660583496 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 13.049027442932129 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 13.056253433227539 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 13.066168785095215 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 13.041265487670898 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 13.032549858093262 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 13.050658226013184 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 13.03137493133545 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 13.055743217468262 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 13.054336547851562 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 13.049675941467285 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 13.060420036315918 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 13.029616355895996 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 13.062712669372559 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 13.030570983886719 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 13.064163208007812 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 13.05705738067627 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 13.062189102172852 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 13.059563636779785 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 13.034463882446289 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 13.056951522827148 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 13.029101371765137 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 13.060152053833008 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 13.057300567626953 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 13.051545143127441 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 13.062098503112793 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 13.025392532348633 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 13.053240776062012 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 13.028526306152344 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 13.063642501831055 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 13.04793930053711 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 13.047802925109863 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 13.056182861328125 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 13.037148475646973 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 13.048698425292969 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 13.021349906921387 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 13.05784797668457 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 13.05808162689209 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 13.057001113891602 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 13.054491996765137 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 13.028918266296387 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 13.045639991760254 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 13.028182029724121 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 13.055662155151367 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 13.06143569946289 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 13.05565071105957 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 13.052151679992676 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 13.033097267150879 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 13.045221328735352 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 13.033075332641602 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 13.057244300842285 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 13.064075469970703 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 13.062954902648926 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 13.067680358886719 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 13.041974067687988 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 13.059398651123047 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 13.035305976867676 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 13.070158004760742 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 13.057792663574219 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 13.052410125732422 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 13.053427696228027 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 13.045764923095703 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 13.04421329498291 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 13.043901443481445 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 13.06135082244873 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 13.059280395507812 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 13.084874153137207 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 13.055121421813965 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 13.039306640625 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 13.042468070983887 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 13.052369117736816 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 13.040509223937988 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 13.05801010131836 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 13.061326026916504 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 13.035066604614258 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 13.062952041625977 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 13.074411392211914 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 13.057586669921875 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 13.049267768859863 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 13.039971351623535 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 13.053409576416016 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 13.047802925109863 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 13.037298202514648 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 13.0574312210083 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 13.04185962677002 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 13.041374206542969 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 13.051180839538574 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 13.051349639892578 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 13.047565460205078 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 13.035026550292969 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 13.035231590270996 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 13.044760704040527 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 13.061020851135254 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 13.067069053649902 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 13.070731163024902 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 13.072245597839355 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 13.043367385864258 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 13.033942222595215 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 13.034234046936035 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 13.090128898620605 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 12.994473457336426 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 13.06491470336914 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 13.013504028320312 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 13.11275863647461 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 13.060267448425293 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 13.00472354888916 |
| } |
| ] |
| }, |
| { |
| "bit": 4, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 9.58368968963623 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 9.58467960357666 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 9.580440521240234 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 9.57874584197998 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 9.574732780456543 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 9.582172393798828 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 9.575645446777344 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 9.584466934204102 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 9.579532623291016 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 9.580706596374512 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 9.576375961303711 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 9.57551383972168 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 9.585015296936035 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 9.580575942993164 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 9.574075698852539 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 9.571317672729492 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 9.582648277282715 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 9.583954811096191 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 9.578856468200684 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 9.57884407043457 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 9.582026481628418 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 9.5790433883667 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 9.57956600189209 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 9.583588600158691 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 9.580179214477539 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 9.576765060424805 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 9.578166961669922 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 9.576703071594238 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 9.578557014465332 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 9.586102485656738 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 9.578678131103516 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 9.58650016784668 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 9.571772575378418 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 9.566763877868652 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 9.570877075195312 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 9.58657169342041 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 9.5878267288208 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 9.595057487487793 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 9.573486328125 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 9.576189041137695 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 9.584802627563477 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 9.575309753417969 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 9.583090782165527 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 9.584096908569336 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 9.57120132446289 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 9.573803901672363 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 9.575766563415527 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 9.580605506896973 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 9.575380325317383 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 9.585211753845215 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 9.584115028381348 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 9.576613426208496 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 9.577385902404785 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 9.568405151367188 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 9.573129653930664 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 9.57342529296875 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 9.588640213012695 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 9.584197044372559 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 9.581779479980469 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 9.594022750854492 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 9.579865455627441 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 9.578737258911133 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 9.58065414428711 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 9.585723876953125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 9.581640243530273 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 9.582767486572266 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 9.578643798828125 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 9.57645320892334 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 9.581679344177246 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 9.571950912475586 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 9.582230567932129 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 9.585735321044922 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 9.580893516540527 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 9.578568458557129 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 9.576929092407227 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 9.585309982299805 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 9.583157539367676 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 9.589150428771973 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 9.579998970031738 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 9.588098526000977 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 9.57970142364502 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 9.584339141845703 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 9.577471733093262 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 9.5828218460083 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 9.580427169799805 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 9.581343650817871 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 9.592024803161621 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 9.565947532653809 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 9.574755668640137 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 9.58171558380127 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 9.580986976623535 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 9.58011245727539 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 9.580998420715332 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 9.581738471984863 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 9.575011253356934 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 9.574139595031738 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 9.589289665222168 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 9.571155548095703 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 9.569360733032227 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 9.566009521484375 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 9.57751750946045 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 9.577122688293457 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 9.576403617858887 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 9.582718849182129 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 9.576411247253418 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 9.584362030029297 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 9.569990158081055 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 9.585709571838379 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 9.577430725097656 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 9.578965187072754 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 9.581873893737793 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 9.577299118041992 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 9.583015441894531 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 9.582571983337402 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 9.581756591796875 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 9.582767486572266 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 9.5734224319458 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 9.580879211425781 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 9.573946952819824 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 9.585426330566406 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 9.584023475646973 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 9.576467514038086 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 9.587026596069336 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 9.577569961547852 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 9.58086109161377 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 9.577326774597168 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 9.584494590759277 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 9.585563659667969 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 9.580806732177734 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 9.581672668457031 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 9.578673362731934 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 9.581670761108398 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 9.57507038116455 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 9.584589958190918 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 9.580304145812988 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 9.582133293151855 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 9.586957931518555 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 9.577367782592773 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 9.580398559570312 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 9.578797340393066 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 9.58255386352539 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 9.585431098937988 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 9.585015296936035 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 9.583252906799316 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 9.57909870147705 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 9.582938194274902 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 9.57776927947998 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 9.582714080810547 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 9.580775260925293 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 9.582746505737305 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 9.581720352172852 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 9.579166412353516 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 9.581975936889648 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 9.577317237854004 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 9.585061073303223 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 9.583849906921387 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 9.582679748535156 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 9.582245826721191 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 9.576380729675293 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 9.584588050842285 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 9.5770845413208 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 9.583905220031738 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 9.585531234741211 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 9.584128379821777 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 9.5820951461792 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 9.577651977539062 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 9.580744743347168 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 9.579171180725098 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 9.58439826965332 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 9.58620834350586 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 9.583828926086426 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 9.584311485290527 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 9.57903003692627 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 9.5830659866333 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 9.579668998718262 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 9.585648536682129 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 9.581844329833984 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 9.583584785461426 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 9.582191467285156 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 9.578065872192383 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 9.577641487121582 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 9.577990531921387 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 9.584452629089355 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 9.584325790405273 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 9.586766242980957 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 9.586427688598633 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 9.578373908996582 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 9.576102256774902 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 9.5807523727417 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 9.580916404724121 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 9.58535385131836 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 9.58250617980957 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 9.586270332336426 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 9.582684516906738 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 9.584512710571289 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 9.578536987304688 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 9.583807945251465 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 9.582019805908203 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 9.583738327026367 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 9.581379890441895 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 9.577508926391602 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 9.580514907836914 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 9.584270477294922 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 9.57970142364502 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 9.586164474487305 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 9.586624145507812 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 9.58095932006836 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 9.582301139831543 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 9.585453987121582 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 9.58311653137207 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 9.585872650146484 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 9.585087776184082 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 9.58742904663086 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 9.585756301879883 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 9.58178424835205 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 9.58070182800293 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 9.578216552734375 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 9.58604621887207 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 9.582011222839355 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 9.58640193939209 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 9.582961082458496 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 9.583719253540039 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 9.59237003326416 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 9.559793472290039 |
| } |
| ] |
| }, |
| { |
| "bit": 5, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.778406143188477 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.777447700500488 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.779881477355957 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.776079177856445 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.778031349182129 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.775751113891602 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.775178909301758 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.77658748626709 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.777767181396484 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.777591705322266 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.777206420898438 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.774002075195312 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.777462005615234 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.776689529418945 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.776951789855957 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.776296615600586 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.778401374816895 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.779263496398926 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.77534008026123 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.778318405151367 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.774453163146973 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.777474403381348 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.777976989746094 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.775001525878906 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.780559539794922 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.774344444274902 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.7772216796875 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.776796340942383 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.778830528259277 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.77869701385498 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.777999877929688 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.777775764465332 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.772913932800293 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.77275276184082 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.777854919433594 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.778979301452637 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.774384498596191 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.775641441345215 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.781765937805176 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.775938034057617 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.780227661132812 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.774049758911133 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.780594825744629 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.777068138122559 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.779412269592285 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.78101634979248 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.776592254638672 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.774290084838867 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.775653839111328 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.777694702148438 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.778154373168945 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.785224914550781 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.7829008102417 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.77574634552002 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.774269104003906 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.772528648376465 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.777868270874023 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.77815055847168 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.778684616088867 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.779923439025879 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.776885986328125 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.777661323547363 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.780928611755371 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.778655052185059 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.778273582458496 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.77935791015625 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.777143478393555 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.778282165527344 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.776110649108887 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.7742338180542 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.777093887329102 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.779685020446777 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.778273582458496 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.77972412109375 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.777543067932129 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.779157638549805 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.777327537536621 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.777491569519043 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.778324127197266 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.777381896972656 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.778169631958008 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.77391529083252 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.78087329864502 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.776725769042969 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.775616645812988 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.777050018310547 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.775091171264648 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.778483390808105 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.776082992553711 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.778600692749023 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.776304244995117 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.778600692749023 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.774917602539062 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.776198387145996 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.7755765914917 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.775080680847168 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.778006553649902 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.776344299316406 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.77680492401123 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.76762580871582 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.775055885314941 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.775625228881836 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.776091575622559 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.778504371643066 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.77372932434082 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.778454780578613 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.768485069274902 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.773797035217285 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.778748512268066 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.776223182678223 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.778509140014648 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.77810001373291 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.777725219726562 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.778265953063965 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.778481483459473 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.781338691711426 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.774619102478027 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.778136253356934 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.776625633239746 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.778306007385254 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.778523445129395 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.775733947753906 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.778860092163086 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.774869918823242 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.776144027709961 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.777863502502441 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.778523445129395 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.777863502502441 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.778639793395996 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.77767562866211 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.776018142700195 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.779090881347656 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.777615547180176 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.779667854309082 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.775691986083984 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.778306007385254 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.780230522155762 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.776361465454102 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.77888298034668 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.774127006530762 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.779196739196777 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.778435707092285 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.778849601745605 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.778772354125977 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.775938034057617 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.778173446655273 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.776731491088867 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.777587890625 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.779818534851074 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.77871322631836 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.779890060424805 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.776871681213379 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.775895118713379 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.776741981506348 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.777691841125488 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.779154777526855 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.778847694396973 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.778868675231934 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.776177406311035 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.7761812210083 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.776966094970703 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.77811050415039 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.77846908569336 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.778206825256348 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.7781400680542 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.775520324707031 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.7780179977417 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.776382446289062 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.778709411621094 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.778554916381836 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.779345512390137 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.778998374938965 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.776545524597168 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.776223182678223 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.777604103088379 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.778119087219238 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.777512550354004 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.779128074645996 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.778240203857422 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.776174545288086 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.777068138122559 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.77651596069336 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.778345108032227 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.778504371643066 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.779935836791992 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.779147148132324 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.777303695678711 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.777006149291992 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.776315689086914 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.7772798538208 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.777524948120117 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.779046058654785 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.777365684509277 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.777355194091797 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.779129981994629 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.777484893798828 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 8.777809143066406 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 8.779004096984863 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 8.778230667114258 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 8.778755187988281 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 8.77780532836914 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 8.778571128845215 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 8.777223587036133 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 8.781336784362793 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 8.779497146606445 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 8.780034065246582 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 8.777717590332031 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 8.778623580932617 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 8.775245666503906 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 8.777462005615234 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 8.778960227966309 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 8.77929973602295 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 8.78048038482666 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 8.7788724899292 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 8.777252197265625 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 8.776972770690918 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 8.775493621826172 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 8.779329299926758 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 8.776165008544922 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 8.77793025970459 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 8.775813102722168 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 8.776302337646484 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 8.767470359802246 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 8.766407012939453 |
| } |
| ] |
| }, |
| { |
| "bit": 6, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.446268081665039 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.445901870727539 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.444463729858398 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.44515323638916 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.445365905761719 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.446763038635254 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.445833206176758 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.446557998657227 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.446335792541504 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.445557594299316 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.445961952209473 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.445184707641602 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.444459915161133 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.446352005004883 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.44485855102539 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.44564437866211 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.443868637084961 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.443276405334473 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.445685386657715 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.443832397460938 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.445220947265625 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.446005821228027 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.446074485778809 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.445877075195312 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.445510864257812 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.4454345703125 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.445760726928711 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.445148468017578 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.44466495513916 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.44526481628418 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.444544792175293 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.445422172546387 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.445068359375 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.446451187133789 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.445416450500488 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.445396423339844 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.449325561523438 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.449925422668457 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.446422576904297 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.445442199707031 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.445683479309082 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.446860313415527 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.446666717529297 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.446847915649414 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.44710636138916 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.443880081176758 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.443854331970215 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.445442199707031 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.44526195526123 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.44786262512207 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.447061538696289 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.444379806518555 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.444355010986328 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.443209648132324 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.442265510559082 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.44590950012207 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.447347640991211 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.447166442871094 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.44661808013916 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.44727897644043 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.445452690124512 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.445460319519043 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.44485092163086 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.44685173034668 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.445730209350586 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.446557998657227 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.448132514953613 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.444609642028809 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.447771072387695 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.446396827697754 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.44629955291748 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.446183204650879 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.447311401367188 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.445523262023926 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.445284843444824 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.445409774780273 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.44580078125 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.446779251098633 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.446304321289062 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.447322845458984 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.44529914855957 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.447250366210938 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.446781158447266 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.445295333862305 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.44620132446289 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.445683479309082 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.443924903869629 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.44295883178711 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.446123123168945 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.445488929748535 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.446090698242188 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.445220947265625 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.445404052734375 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.44498348236084 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.444526672363281 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.44593334197998 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.446650505065918 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.444852828979492 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.444454193115234 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.442167282104492 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.445760726928711 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.444586753845215 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.444647789001465 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.445494651794434 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.4448823928833 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.446949005126953 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.441555976867676 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.447705268859863 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.445769309997559 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.44493293762207 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.445792198181152 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.442906379699707 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.446367263793945 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.446304321289062 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.445438385009766 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.444693565368652 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.445124626159668 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.44657039642334 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.444841384887695 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.446453094482422 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.447112083435059 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.446263313293457 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.445061683654785 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.445284843444824 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.446332931518555 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.444198608398438 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.44651985168457 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.446598052978516 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.446005821228027 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.445953369140625 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.445769309997559 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.445100784301758 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.444735527038574 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.446372985839844 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.445813179016113 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.445708274841309 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.446630477905273 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.444903373718262 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.445507049560547 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.446324348449707 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.446215629577637 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.447033882141113 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.44589900970459 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.446724891662598 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.445246696472168 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.445833206176758 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.445683479309082 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.446892738342285 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.445212364196777 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.445100784301758 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.445877075195312 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.444957733154297 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.446231842041016 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.444456100463867 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.446710586547852 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.446854591369629 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.44576644897461 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.44590950012207 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.445233345031738 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.446449279785156 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.44591999053955 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.446203231811523 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.446487426757812 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.446499824523926 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.446240425109863 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.445124626159668 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.444564819335938 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.445398330688477 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.446983337402344 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.447197914123535 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.446784019470215 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.447357177734375 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.444705963134766 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.445945739746094 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.444546699523926 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.446733474731445 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.44636058807373 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.446362495422363 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.447315216064453 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.445897102355957 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.444729804992676 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.445178031921387 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.446642875671387 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.446335792541504 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.44752025604248 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.446934700012207 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.445253372192383 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.446026802062988 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.444671630859375 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.446354866027832 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.445853233337402 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.446110725402832 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.446598052978516 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.446369171142578 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.445677757263184 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.446175575256348 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 8.446813583374023 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 8.446487426757812 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 8.446715354919434 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 8.446258544921875 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 8.445032119750977 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 8.44625186920166 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 8.446349143981934 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 8.445685386657715 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 8.446003913879395 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 8.445815086364746 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 8.4470853805542 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 8.445157051086426 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 8.44617748260498 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 8.444929122924805 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 8.446699142456055 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 8.446615219116211 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 8.447568893432617 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 8.44766902923584 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 8.445507049560547 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 8.444254875183105 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 8.44710636138916 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 8.446497917175293 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 8.447943687438965 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 8.44632625579834 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 8.44657039642334 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 8.446557998657227 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 8.445124626159668 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 8.442789077758789 |
| } |
| ] |
| }, |
| { |
| "bit": 7, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.248947143554688 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.248910903930664 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.25033950805664 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.247231483459473 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.248175621032715 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.249284744262695 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.248127937316895 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.24847412109375 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.24928092956543 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.249184608459473 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.249013900756836 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.248451232910156 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.249009132385254 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.24847412109375 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.247840881347656 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.248675346374512 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.250819206237793 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.249013900756836 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.248690605163574 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.24866008758545 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.248364448547363 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.248971939086914 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.24841594696045 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.246623992919922 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.248726844787598 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.246759414672852 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.247727394104004 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.247196197509766 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.249139785766602 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.249351501464844 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.250386238098145 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.249241828918457 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.246981620788574 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.248429298400879 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.250593185424805 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.250153541564941 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.249406814575195 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.248666763305664 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.248573303222656 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.247270584106445 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.246830940246582 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.2464017868042 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.248993873596191 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.248223304748535 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.248842239379883 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.250232696533203 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.247780799865723 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.247939109802246 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.24886417388916 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.248420715332031 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.248710632324219 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.25039291381836 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.250648498535156 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.248642921447754 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.248636245727539 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.246983528137207 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.24928092956543 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.248407363891602 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.250875473022461 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.24842357635498 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.248676300048828 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.248442649841309 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.249731063842773 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.248982429504395 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.248353958129883 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.248805046081543 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.247725486755371 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.247814178466797 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.248486518859863 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.24753475189209 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.249300003051758 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.249649047851562 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.247969627380371 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.249044418334961 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.248712539672852 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.24815845489502 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.249027252197266 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.24854564666748 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.248103141784668 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.249629020690918 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.24824047088623 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.2481050491333 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.247129440307617 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.248519897460938 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.248733520507812 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.248607635498047 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.249269485473633 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.249617576599121 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.247703552246094 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.248675346374512 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.24885368347168 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.248747825622559 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.248342514038086 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.24862003326416 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.247286796569824 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.247939109802246 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.248085021972656 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.248418807983398 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.248696327209473 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.244585037231445 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.248316764831543 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.24746322631836 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.248014450073242 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.248686790466309 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.248486518859863 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.248899459838867 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.24615478515625 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.248085021972656 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.248323440551758 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.248067855834961 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.248703002929688 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.248798370361328 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.248700141906738 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.248745918273926 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.247742652893066 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.249890327453613 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.246377944946289 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.248982429504395 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.247485160827637 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.248905181884766 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.248494148254395 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.248453140258789 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.248908996582031 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.248163223266602 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.248038291931152 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.248063087463379 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.24870491027832 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.248576164245605 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.248796463012695 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.24866008758545 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.247627258300781 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.249587059020996 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.248126983642578 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.249279022216797 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.247621536254883 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.248908996582031 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.248923301696777 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.24789810180664 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.24841594696045 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.248531341552734 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.249009132385254 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.24869441986084 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.24914264678955 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.248514175415039 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.247979164123535 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.249058723449707 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.2473726272583 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.248682975769043 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.249214172363281 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.248796463012695 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.249458312988281 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.248420715332031 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.24854564666748 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.24852180480957 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.248820304870605 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.248818397521973 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.248915672302246 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.248651504516602 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.247900009155273 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.248698234558105 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.248372077941895 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.248966217041016 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.248855590820312 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.248828887939453 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.248640060424805 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.2474365234375 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.248297691345215 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.248397827148438 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.249364852905273 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.249395370483398 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.248762130737305 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.248749732971191 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.249072074890137 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.2484712600708 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.248058319091797 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.248920440673828 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.249072074890137 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.249566078186035 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.248096466064453 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.248063087463379 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.248846054077148 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.247840881347656 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.248753547668457 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.248805046081543 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.248598098754883 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.24842357635498 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.248464584350586 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.247590065002441 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.248529434204102 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.248451232910156 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.248435020446777 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.248573303222656 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.248891830444336 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.248395919799805 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.249190330505371 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.247916221618652 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 8.248773574829102 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 8.24884033203125 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 8.248529434204102 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 8.248580932617188 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 8.247758865356445 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 8.248266220092773 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 8.249462127685547 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 8.251774787902832 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 8.249398231506348 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 8.249236106872559 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 8.249151229858398 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 8.24839973449707 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 8.246936798095703 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 8.248584747314453 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 8.24909782409668 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 8.249333381652832 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 8.24841594696045 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 8.248838424682617 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 8.24821662902832 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 8.24967098236084 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 8.246960639953613 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 8.248788833618164 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 8.248041152954102 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 8.248600006103516 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 8.248079299926758 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 8.249293327331543 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 8.249005317687988 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 8.244156837463379 |
| } |
| ] |
| }, |
| { |
| "bit": 8, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.151074409484863 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.150999069213867 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.15017318725586 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.150079727172852 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.149909019470215 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.151076316833496 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.149611473083496 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.151737213134766 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.150666236877441 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.149551391601562 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.150126457214355 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.150810241699219 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.150899887084961 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.150544166564941 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.150005340576172 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.150588989257812 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.148330688476562 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.152186393737793 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.1503324508667 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.149892807006836 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.14985179901123 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.151497840881348 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.15092658996582 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.152995109558105 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.14924430847168 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.152137756347656 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.149636268615723 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.151265144348145 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.15049934387207 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.150126457214355 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.150693893432617 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.151034355163574 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.150919914245605 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.15013599395752 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.149358749389648 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.150907516479492 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.151817321777344 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.152956008911133 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.151284217834473 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.150101661682129 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.150654792785645 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.151396751403809 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.151276588439941 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.15075969696045 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.150779724121094 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.150075912475586 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.150604248046875 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.151717185974121 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.150860786437988 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.15180492401123 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.15007209777832 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.149370193481445 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.149572372436523 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.150145530700684 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.150029182434082 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.151187896728516 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.15129280090332 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.151187896728516 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.15044116973877 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.150922775268555 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.15063190460205 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.151873588562012 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.150771141052246 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.151354789733887 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.150899887084961 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.151061058044434 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.150907516479492 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.150270462036133 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.151175498962402 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.151531219482422 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.150693893432617 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.151203155517578 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.150079727172852 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.150582313537598 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.150604248046875 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.150514602661133 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.149866104125977 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.151334762573242 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.150800704956055 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.15058708190918 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.150219917297363 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.150744438171387 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.152250289916992 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.150511741638184 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.150794982910156 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.1510009765625 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.15099048614502 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.149415016174316 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.151323318481445 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.150444984436035 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.150690078735352 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.15068244934082 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.150378227233887 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.150261878967285 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.150903701782227 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.15018081665039 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.15124225616455 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.150435447692871 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.150575637817383 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.149094581604004 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.15013599395752 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.151214599609375 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.149642944335938 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.150808334350586 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.150075912475586 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.151111602783203 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.148834228515625 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.14988899230957 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.150835037231445 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.150402069091797 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.15018081665039 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.150197982788086 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.150830268859863 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.151409149169922 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.151313781738281 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.150514602661133 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.15072250366211 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.150216102600098 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.15139389038086 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.151105880737305 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.15129280090332 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.150542259216309 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.150880813598633 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.150386810302734 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.150701522827148 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.150864601135254 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.150973320007324 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.151409149169922 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.150779724121094 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.150169372558594 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.150347709655762 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.1502046585083 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.14988899230957 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.150887489318848 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.150087356567383 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.150806427001953 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.150779724121094 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.149798393249512 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.150814056396484 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.149948120117188 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.151045799255371 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.15125560760498 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.150981903076172 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.151280403137207 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.150588989257812 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.150777816772461 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.150565147399902 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.150872230529785 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.150402069091797 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.150662422180176 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.150643348693848 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.149776458740234 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.150899887084961 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.150068283081055 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.150911331176758 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.151187896728516 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.15097713470459 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.151018142700195 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.15003490447998 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.150360107421875 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.150002479553223 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.151078224182129 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.150967597961426 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.151137351989746 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.15079116821289 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.15003490447998 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.150252342224121 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.150211334228516 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.150823593139648 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.150959968566895 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.151315689086914 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.151352882385254 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.149739265441895 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.150032997131348 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.150887489318848 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.150884628295898 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.150869369506836 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.150604248046875 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.151308059692383 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.150044441223145 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.150025367736816 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.150693893432617 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.15110969543457 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.150965690612793 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.15166187286377 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.150911331176758 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.15001392364502 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.150763511657715 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.150779724121094 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.150981903076172 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.151012420654297 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.151111602783203 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.150527000427246 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.150915145874023 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.150227546691895 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.150995254516602 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 8.151082038879395 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 8.150897979736328 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 8.150961875915527 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 8.150580406188965 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 8.15087604522705 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 8.150973320007324 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 8.149886131286621 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 8.150259971618652 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 8.150981903076172 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 8.151164054870605 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 8.1516695022583 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 8.150505065917969 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 8.151898384094238 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 8.150216102600098 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 8.150959968566895 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 8.151091575622559 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 8.151031494140625 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 8.151062965393066 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 8.151004791259766 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 8.149847030639648 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 8.151618957519531 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 8.150933265686035 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 8.150798797607422 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 8.150670051574707 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 8.150973320007324 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 8.151222229003906 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 8.150849342346191 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 8.149383544921875 |
| } |
| ] |
| }, |
| { |
| "bit": 9, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.067623138427734 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.067371368408203 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.067314147949219 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.067713737487793 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.068294525146484 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.066909790039062 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.066911697387695 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.067405700683594 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.067975044250488 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.067959785461426 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.06697940826416 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.067327499389648 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.068471908569336 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.067337036132812 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.067325592041016 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.0684814453125 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.068489074707031 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.067191123962402 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.067529678344727 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.068572044372559 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.067832946777344 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.067422866821289 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.066040992736816 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.065824508666992 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.068029403686523 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.067051887512207 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.069037437438965 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.068551063537598 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.067594528198242 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.0676908493042 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.066457748413086 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.067981719970703 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.067009925842285 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.067184448242188 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.067936897277832 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.068052291870117 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.067959785461426 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.06750774383545 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.067174911499023 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.067588806152344 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.066601753234863 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.064970970153809 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.06764030456543 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.067378997802734 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.067084312438965 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.067852973937988 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.066695213317871 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.066423416137695 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.067400932312012 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.067340850830078 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.067607879638672 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.068846702575684 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.068791389465332 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.066926956176758 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.066832542419434 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.066413879394531 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.067556381225586 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.066998481750488 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.0679292678833 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.069302558898926 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.066902160644531 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.066278457641602 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.068281173706055 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.067770004272461 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.067181587219238 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.06659984588623 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.066482543945312 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.067448616027832 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.066686630249023 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.066936492919922 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.067852973937988 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.067567825317383 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.067138671875 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.068078994750977 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.06711196899414 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.067625045776367 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.067371368408203 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.067009925842285 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.067075729370117 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.066960334777832 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.067917823791504 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.06688117980957 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.066499710083008 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.067252159118652 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.06723690032959 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.067049980163574 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.066977500915527 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.066834449768066 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.066579818725586 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.067399024963379 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.06697940826416 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.06728458404541 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.067429542541504 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.066926956176758 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.06646728515625 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.066909790039062 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.067216873168945 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.067548751831055 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.066932678222656 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.0663480758667 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.067214012145996 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.066629409790039 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.066947937011719 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.067144393920898 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.067333221435547 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.066960334777832 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.066656112670898 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.067325592041016 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.066917419433594 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.06721019744873 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.067075729370117 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.067727088928223 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.067562103271484 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.06740951538086 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.066727638244629 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.067598342895508 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.065759658813477 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.067893028259277 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.06623649597168 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.067214012145996 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.067073822021484 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.06689453125 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.067201614379883 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.06643295288086 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.066864967346191 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.066125869750977 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.067594528198242 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.067240715026855 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.067051887512207 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.067164421081543 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.066367149353027 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.067806243896484 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.067405700683594 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.067583084106445 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.06662368774414 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.06707763671875 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.06748104095459 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.066814422607422 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.067150115966797 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.06710433959961 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.06735610961914 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.067261695861816 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.06727123260498 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.067252159118652 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.0669584274292 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.06726360321045 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.066512107849121 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.067444801330566 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.067535400390625 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.06706714630127 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.06736946105957 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.067084312438965 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.06692123413086 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.067275047302246 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.06718635559082 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.067146301269531 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.067079544067383 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.066890716552734 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.067407608032227 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.067298889160156 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.067463874816895 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.06735610961914 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.067179679870605 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.067066192626953 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.067422866821289 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.066814422607422 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.066925048828125 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.067325592041016 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.067655563354492 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.067460060119629 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.066898345947266 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.067273139953613 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.067323684692383 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.06699275970459 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.0665864944458 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.067240715026855 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.06734848022461 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.067789077758789 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.066925048828125 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.067155838012695 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.06695556640625 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.066548347473145 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.067193031311035 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.067556381225586 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.067394256591797 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.067628860473633 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.066964149475098 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.067400932312012 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.066832542419434 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.067021369934082 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.067121505737305 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.067450523376465 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.067978858947754 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.06725025177002 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.067359924316406 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.066597938537598 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 8.067267417907715 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 8.067283630371094 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 8.067208290100098 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 8.06723690032959 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 8.066777229309082 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 8.066756248474121 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 8.067429542541504 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 8.067161560058594 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 8.06722354888916 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 8.067436218261719 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 8.066812515258789 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 8.067388534545898 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 8.066712379455566 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 8.067301750183105 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 8.067394256591797 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 8.067215919494629 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 8.067553520202637 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 8.067460060119629 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 8.066627502441406 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 8.06711196899414 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 8.0663480758667 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 8.067488670349121 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 8.066911697387695 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 8.06745433807373 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 8.067044258117676 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 8.067448616027832 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 8.066147804260254 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 8.064809799194336 |
| } |
| ] |
| }, |
| { |
| "bit": 10, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.016072273254395 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.01572036743164 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.015779495239258 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.013850212097168 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.015554428100586 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.016152381896973 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.01573371887207 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.016106605529785 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.015592575073242 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.014792442321777 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.015885353088379 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.015843391418457 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.01540756225586 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.016087532043457 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.01546859741211 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.016053199768066 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.01502513885498 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.017158508300781 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.015349388122559 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.015369415283203 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.015707015991211 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.015777587890625 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.015740394592285 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.017233848571777 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.016221046447754 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.01547622680664 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.013744354248047 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.015820503234863 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.015275955200195 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.014629364013672 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.015417098999023 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.015705108642578 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.01575756072998 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.015128135681152 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.015640258789062 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.015661239624023 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.015885353088379 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.015984535217285 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.014740943908691 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.014925956726074 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.014516830444336 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.016642570495605 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.016022682189941 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.015360832214355 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.01651382446289 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.015082359313965 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.015556335449219 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.016702651977539 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.014656066894531 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.016011238098145 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.01544189453125 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.014313697814941 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.014848709106445 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.015477180480957 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.015109062194824 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.01556396484375 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.015725135803223 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.015539169311523 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.01572036743164 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.014037132263184 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.015162467956543 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.016515731811523 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.01517391204834 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.015525817871094 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.014971733093262 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.016302108764648 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.016420364379883 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.015124320983887 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.015430450439453 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.015632629394531 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.015360832214355 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.015809059143066 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.015645980834961 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.015340805053711 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.015360832214355 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.015401840209961 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.015289306640625 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.01595401763916 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.015902519226074 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.015829086303711 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.015710830688477 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.015657424926758 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.01676082611084 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.015522003173828 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.015560150146484 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.01569938659668 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.015657424926758 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.01547622680664 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.015851020812988 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.01551628112793 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.015908241271973 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.01565170288086 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.015670776367188 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.01548957824707 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.014751434326172 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.0158052444458 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.015727996826172 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.015722274780273 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.015774726867676 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.014805793762207 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.015362739562988 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.01598072052002 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.015769958496094 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.015583038330078 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.015472412109375 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.015750885009766 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.015069007873535 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.015493392944336 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.01559066772461 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.015027046203613 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.015701293945312 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.014756202697754 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.015510559082031 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.01571273803711 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.015560150146484 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.01537799835205 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.015793800354004 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.015349388122559 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.015571594238281 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.015885353088379 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.015772819519043 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.01509952545166 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.015621185302734 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.015426635742188 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.015628814697266 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.015556335449219 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.015602111816406 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.015594482421875 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.015811920166016 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.015523910522461 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.015731811523438 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.015135765075684 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.01469612121582 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.01568603515625 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.015399932861328 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.0157470703125 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.015573501586914 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.015280723571777 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.015426635742188 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.015299797058105 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.01565170288086 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.015868186950684 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.015632629394531 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.015829086303711 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.01514720916748 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.015705108642578 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.015910148620605 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.015628814697266 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.015725135803223 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.015820503234863 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.015647888183594 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.015191078186035 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.015379905700684 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.015149116516113 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.015727996826172 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.016009330749512 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.015762329101562 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.015332221984863 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.015636444091797 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.015419006347656 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.014669418334961 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.01570987701416 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.015827178955078 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.015898704528809 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.01575756072998 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.01526165008545 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.014975547790527 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.015481948852539 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.0158052444458 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.01585865020752 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.015950202941895 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.015764236450195 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.015149116516113 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.01553726196289 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.015166282653809 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.016026496887207 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.015839576721191 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.015560150146484 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.015844345092773 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.015491485595703 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.014870643615723 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.015875816345215 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.015854835510254 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.01578140258789 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.015820503234863 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.015586853027344 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.015535354614258 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.015332221984863 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.01523494720459 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.015959739685059 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.015628814697266 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.015625 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.015480041503906 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.015525817871094 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.015263557434082 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.016241073608398 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 8.015740394592285 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 8.015807151794434 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 8.01569938659668 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 8.015548706054688 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 8.01513957977295 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 8.015565872192383 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 8.015539169311523 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 8.015677452087402 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 8.015495300292969 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 8.01573371887207 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 8.015873908996582 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 8.014962196350098 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 8.015810012817383 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 8.014935493469238 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 8.015657424926758 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 8.015716552734375 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 8.016037940979004 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 8.01623249053955 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 8.0159330368042 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 8.016399383544922 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 8.015539169311523 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 8.015666961669922 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 8.015567779541016 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 8.015960693359375 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 8.015499114990234 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 8.015135765075684 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 8.013542175292969 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 8.014952659606934 |
| } |
| ] |
| }, |
| { |
| "bit": 11, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.964605331420898 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.964717864990234 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.963918685913086 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.964454174041748 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.965520858764648 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.964694976806641 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.964081764221191 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.964750289916992 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.964765548706055 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.965912818908691 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.964085578918457 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.964970111846924 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.965302467346191 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.964750289916992 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.964552879333496 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.965578079223633 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.964978218078613 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.964252948760986 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.965638637542725 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.965236186981201 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.964364528656006 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.965023517608643 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.964137077331543 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.964049339294434 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.964387893676758 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.964309215545654 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.96549654006958 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.964499473571777 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.965106964111328 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.965099811553955 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.964205265045166 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.964746475219727 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.964237689971924 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.965942859649658 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.964859962463379 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.9645562171936035 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.965295314788818 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.965863227844238 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.965365886688232 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.9650163650512695 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.964442253112793 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.963918685913086 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.964932441711426 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.964480400085449 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.965057849884033 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.965120315551758 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.96401309967041 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.964643478393555 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.9651641845703125 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.9651713371276855 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.964643478393555 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.965460300445557 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.965175628662109 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.964601993560791 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.964146137237549 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.964431285858154 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.964890480041504 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.964850902557373 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.96538782119751 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.966098308563232 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.965152263641357 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.964096546173096 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.9650044441223145 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.964999198913574 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.964670658111572 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.964389801025391 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.964089393615723 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.9645256996154785 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.964934349060059 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.964175224304199 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.964991092681885 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.964664459228516 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.964712142944336 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.964696884155273 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.964609622955322 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.964184284210205 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.964624404907227 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.964731216430664 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.964516639709473 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.964993000030518 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.964263916015625 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.964488506317139 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.964402675628662 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.964738368988037 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.964560031890869 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.964544773101807 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.964890480041504 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.963897228240967 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.964389801025391 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.965316295623779 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.964731216430664 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.964503288269043 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.964643478393555 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.96470832824707 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.964254856109619 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.964338302612305 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.964414119720459 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.965084552764893 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.964207172393799 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.9640607833862305 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.964677810668945 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.964433193206787 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.9643707275390625 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.964776515960693 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.964890480041504 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.964840888977051 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.964601993560791 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.964417934417725 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.964118003845215 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.964818477630615 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.964771270751953 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.964954853057861 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.96466064453125 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.96486234664917 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.9643707275390625 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.964643478393555 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.96377420425415 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.964677810668945 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.964156150817871 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.964670658111572 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.964609622955322 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.964207172393799 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.964176177978516 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.964544773101807 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.964402675628662 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.963929653167725 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.9649739265441895 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.96478796005249 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.964738368988037 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.964704990386963 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.9644975662231445 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.964943885803223 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.964776515960693 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.964818477630615 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.964598178863525 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.9644694328308105 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.964754104614258 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.964231491088867 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.964653491973877 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.964601993560791 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.964852809906006 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.964600086212158 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.96473503112793 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.964757442474365 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.964710235595703 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.964892387390137 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.96381950378418 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.964693069458008 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.964879512786865 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.964605331420898 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.964568138122559 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.964784622192383 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.9649176597595215 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.965052604675293 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.964636325836182 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.9647955894470215 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.964563846588135 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.964765548706055 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.964274883270264 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.965055465698242 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.964712142944336 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.9648871421813965 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.96493673324585 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.964651584625244 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.964710235595703 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.964175224304199 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.964510917663574 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.964036464691162 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.964943885803223 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.964840888977051 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.964560031890869 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.964704990386963 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.964919567108154 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.9641499519348145 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.96450138092041 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.964621067047119 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.964864253997803 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.964890480041504 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.964803695678711 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.964305877685547 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.96493673324585 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.9647955894470215 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.9645490646362305 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.964954853057861 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.96500301361084 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.964771270751953 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.964423656463623 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.964552879333496 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.964328289031982 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.964636325836182 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.964684009552002 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.9645490646362305 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.9649739265441895 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.964776515960693 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.964719772338867 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.964321136474609 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.964856147766113 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.964503288269043 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.965101718902588 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.964563846588135 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.964305877685547 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.965035438537598 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.964750289916992 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.96473503112793 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.964820861816406 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.96476936340332 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.964812755584717 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.964561939239502 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.964465141296387 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.964831829071045 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.964761257171631 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.964622497558594 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.964542865753174 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.964523792266846 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.96421480178833 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.964547157287598 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.964442253112793 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.96466064453125 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.96444034576416 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.96466064453125 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.964396953582764 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.96507453918457 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.963815689086914 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.962518692016602 |
| } |
| ] |
| }, |
| { |
| "bit": 12, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.934129238128662 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.93372106552124 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.936017990112305 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.933906555175781 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.933186054229736 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.93415641784668 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.933126926422119 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.934360027313232 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.9343109130859375 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.933248519897461 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.93372106552124 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.93415641784668 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.933910369873047 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.933342933654785 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.933637619018555 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.934277057647705 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.9337592124938965 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.93411111831665 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.934088230133057 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.9338154792785645 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.933320045471191 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.933942794799805 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.934088230133057 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.934545516967773 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.93414831161499 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.93388557434082 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.933516979217529 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.9342546463012695 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.933891296386719 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.933448791503906 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.934754371643066 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.934265613555908 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.93405818939209 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.932851314544678 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.934825897216797 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.934067249298096 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.933448791503906 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.933569431304932 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.93362283706665 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.932837963104248 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.933821201324463 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.934160232543945 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.934116363525391 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.934638977050781 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.933763027191162 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.933970928192139 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.934314727783203 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.93358850479126 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.9337968826293945 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.934328079223633 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.93355655670166 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.933483123779297 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.9330668449401855 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.933656692504883 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.9337286949157715 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.934144973754883 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.93381404876709 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.93385124206543 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.9332404136657715 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.933043956756592 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.9338483810424805 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.934352874755859 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.93358850479126 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.933749198913574 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.93378210067749 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.934314727783203 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.93448543548584 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.93405818939209 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.933607578277588 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.934072971343994 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.934086322784424 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.934205532073975 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.933509349822998 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.933671951293945 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.933773994445801 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.933562278747559 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.933722972869873 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.934152603149414 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.93407678604126 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.933411121368408 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.933831214904785 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.934088230133057 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.93415641784668 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.933518409729004 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.933898448944092 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.934039115905762 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.933770179748535 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.93420934677124 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.934046268463135 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.933117866516113 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.934055805206299 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.934025764465332 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.934263706207275 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.933863162994385 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.9343037605285645 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.933955669403076 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.934280872344971 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.934051990509033 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.934103012084961 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.933267593383789 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.933906555175781 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.933857440948486 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.9337477684021 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.93408203125 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.9338765144348145 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.93388557434082 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.933732032775879 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.933959484100342 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.934375762939453 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.933607578277588 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.9340009689331055 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.933936595916748 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.933838367462158 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.934141159057617 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.9336419105529785 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.933942794799805 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.934241771697998 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.933488368988037 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.934354782104492 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.93415641784668 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.934002876281738 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.93383264541626 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.934013843536377 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.93349027633667 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.933881759643555 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.933910369873047 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.933732032775879 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.934247016906738 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.933902740478516 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.934067249298096 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.934109210968018 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.933754920959473 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.933513164520264 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.9339728355407715 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.934103012084961 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.933857440948486 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.933985710144043 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.93355655670166 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.934025764465332 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.933645725250244 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.934025764465332 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.934146404266357 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.934042930603027 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.933993816375732 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.933691024780273 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.933630466461182 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.9337358474731445 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.9341583251953125 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.9338765144348145 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.934021949768066 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.933887481689453 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.933367729187012 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.934190273284912 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.933584690093994 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.934046268463135 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.934051990509033 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.934093952178955 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.93409538269043 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.933821201324463 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.9339728355407715 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.93345832824707 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.933955669403076 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.933974742889404 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.934210777282715 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.933963775634766 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.933724880218506 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.933615684509277 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.934122085571289 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.934250831604004 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.934013843536377 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.9343719482421875 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.9341864585876465 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.933637619018555 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.9338836669921875 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.934178352355957 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.934356689453125 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.934118270874023 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.93405818939209 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.934430122375488 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.933948040008545 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.9332404136657715 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.933861255645752 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.9340128898620605 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.9339447021484375 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.933919429779053 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.933906555175781 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.933831214904785 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.934034824371338 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.933913707733154 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.934175491333008 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.933849334716797 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.934292793273926 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.93391752243042 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.933679580688477 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.933710098266602 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.934432506561279 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.934114456176758 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.934210777282715 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.933777809143066 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.934093952178955 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.933834552764893 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.93364953994751 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.933694839477539 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.934201717376709 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.934079170227051 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.934231758117676 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.933955669403076 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.933682918548584 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.933953762054443 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.933509349822998 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.934273719787598 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.934116363525391 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.934192180633545 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.934500694274902 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.934457302093506 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.933518409729004 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.933679580688477 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.934322834014893 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.933902740478516 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.93407678604126 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.934103012084961 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.933732032775879 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.933861255645752 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.933682918548584 |
| } |
| ] |
| }, |
| { |
| "bit": 13, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.8956828117370605 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.89573335647583 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.894374370574951 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.8953046798706055 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.89564323425293 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.895830154418945 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.896002769470215 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.895630359649658 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.895513534545898 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.8961687088012695 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.896307468414307 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.895694255828857 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.896726131439209 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.895764350891113 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.895650863647461 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.895998954772949 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.896126747131348 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.895295143127441 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.895564556121826 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.896631717681885 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.8966169357299805 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.895965099334717 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.895095348358154 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.894772052764893 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.895986080169678 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.895052433013916 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.89622163772583 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.895277976989746 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.895852088928223 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.895869255065918 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.895908832550049 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.896046161651611 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.895775318145752 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.896175861358643 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.895323276519775 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.895368576049805 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.896428108215332 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.896627902984619 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.895649433135986 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.8954010009765625 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.895233154296875 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.895171165466309 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.8955607414245605 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.895434856414795 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.895754337310791 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.8963117599487305 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.895739555358887 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.896117687225342 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.895961761474609 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.895698070526123 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.895498752593994 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.896678924560547 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.8964362144470215 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.8955206871032715 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.895924091339111 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.895174503326416 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.895956039428711 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.895362854003906 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.896251678466797 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.897162437438965 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.895675182342529 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.895190238952637 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.895645618438721 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.89582633972168 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.895641326904297 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.895571708679199 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.895076274871826 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.895146369934082 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.895658493041992 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.895318031311035 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.895584583282471 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.895481586456299 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.895583629608154 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.895359039306641 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.895360946655273 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.895660400390625 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.895846843719482 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.895596504211426 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.8954925537109375 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.89573335647583 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.895809173583984 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.895403861999512 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.895298957824707 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.895803451538086 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.895381927490234 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.8954267501831055 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.895140647888184 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.894763946533203 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.894922733306885 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.8962249755859375 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.895295143127441 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.895748615264893 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.895641326904297 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.895782470703125 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.8951334953308105 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.895419597625732 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.895370960235596 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.895479679107666 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.895403861999512 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.89509916305542 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.895415782928467 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.895652770996094 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.89562463760376 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.89539098739624 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.895415782928467 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.895551681518555 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.895366668701172 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.895396709442139 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.895470142364502 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.896060943603516 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.895468711853027 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.895201206207275 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.895799160003662 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.895442008972168 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.895609378814697 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.895637512207031 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.894946575164795 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.896158695220947 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.894984722137451 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.895528793334961 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.895724296569824 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.895240306854248 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.8952460289001465 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.895235061645508 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.895737171173096 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.894929885864258 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.895430564880371 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.895541667938232 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.895357131958008 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.895231246948242 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.894967555999756 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.895703315734863 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.89561128616333 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.895503997802734 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.895522594451904 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.895366668701172 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.8955206871032715 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.895242214202881 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.895577430725098 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.895434856414795 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.895660400390625 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.895517349243164 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.895600318908691 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.895502090454102 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.895242214202881 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.895750522613525 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.895174503326416 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.895199298858643 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.89581823348999 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.895662307739258 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.895430564880371 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.895660400390625 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.895056247711182 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.895852088928223 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.895652770996094 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.895622730255127 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.8951520919799805 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.895498752593994 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.89562463760376 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.8953399658203125 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.8961310386657715 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.895577430725098 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.895566463470459 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.895176410675049 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.895370960235596 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.894986629486084 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.895667552947998 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.895009517669678 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.895488739013672 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.895630359649658 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.895308017730713 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.895758152008057 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.89570951461792 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.895498752593994 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.895359039306641 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.895628452301025 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.895605564117432 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.895637512207031 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.895543575286865 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.895242214202881 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.895820140838623 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.895549297332764 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.895566463470459 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.895761966705322 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.895822048187256 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.895502090454102 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.895486831665039 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.895569801330566 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.895860195159912 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.895483493804932 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.895901203155518 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.895679473876953 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.8956074714660645 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.8958940505981445 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.895456790924072 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.895295143127441 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.8954267501831055 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.895562648773193 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.89567756652832 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.895632266998291 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.895359039306641 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.89578104019165 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.895551681518555 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.896018028259277 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.895411968231201 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.895556926727295 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.895667552947998 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.895573616027832 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.895295143127441 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.89584493637085 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.895503997802734 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.895509719848633 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.895528793334961 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.895442008972168 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.895163536071777 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.895954132080078 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.895790100097656 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.895615577697754 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.895419597625732 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.895901203155518 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.895490646362305 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.896251678466797 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.895005226135254 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.894482135772705 |
| } |
| ] |
| }, |
| { |
| "bit": 14, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.880334854125977 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.88006067276001 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.880678653717041 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.880212783813477 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.879787921905518 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.880086898803711 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.879388332366943 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.880111217498779 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.880492687225342 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.879292011260986 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.879976272583008 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.880184650421143 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.879919528961182 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.880273342132568 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.88018274307251 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.880095958709717 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.879842758178711 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.880460739135742 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.880008220672607 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.879767417907715 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.8791961669921875 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.8802971839904785 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.879979610443115 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.881012916564941 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.880107879638672 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.880434513092041 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.8791890144348145 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.880094051361084 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.879912376403809 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.880113124847412 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.880224227905273 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.8803181648254395 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.879900932312012 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.879220962524414 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.879660606384277 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.880471706390381 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.880030155181885 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.8806939125061035 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.879403114318848 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.880186557769775 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.8806352615356445 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.880791664123535 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.880276203155518 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.8806538581848145 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.880389213562012 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.879976272583008 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.879840850830078 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.880631923675537 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.879641532897949 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.880451679229736 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.8797502517700195 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.878563404083252 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.8796305656433105 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.879915714263916 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.8796844482421875 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.879904747009277 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.88001012802124 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.880090713500977 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.8796563148498535 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.879581451416016 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.8802056312561035 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.880757808685303 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.880280494689941 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.87990665435791 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.88039493560791 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.880056858062744 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.880690097808838 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.8799028396606445 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.8799614906311035 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.880396842956543 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.879998207092285 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.8802900314331055 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.879422187805176 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.880302906036377 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.88018274307251 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.879996299743652 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.8795270919799805 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.8802900314331055 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.880186557769775 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.879912376403809 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.879652500152588 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.880186557769775 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.880644798278809 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.879812717437744 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.880430698394775 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.880173683166504 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.880256175994873 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.880690097808838 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.880320072174072 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.879589080810547 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.880148887634277 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.880032062530518 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.880227565765381 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.880280494689941 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.879776954650879 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.880086898803711 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.880116939544678 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.880184650421143 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.880321979522705 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.8799872398376465 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.880113124847412 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.880111217498779 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.880088806152344 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.880058765411377 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.8798747062683105 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.880083084106445 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.879889488220215 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.880099773406982 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.880468368530273 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.8800249099731445 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.880207538604736 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.880359172821045 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.879851818084717 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.880280494689941 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.879707336425781 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.879946231842041 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.880173683166504 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.8800740242004395 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.880175590515137 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.880210876464844 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.879929542541504 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.880163669586182 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.880273342132568 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.88009786605835 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.880107879638672 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.8802056312561035 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.880302906036377 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.880299091339111 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.880077838897705 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.880596160888672 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.880116939544678 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.880154609680176 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.879668235778809 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.880242347717285 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.8803181648254395 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.880197525024414 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.880141735076904 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.879796028137207 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.880192279815674 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.8799614906311035 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.880320072174072 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.880332946777344 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.880307197570801 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.8803629875183105 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.87985372543335 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.880141735076904 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.880126476287842 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.880302906036377 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.879964351654053 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.880246639251709 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.880133628845215 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.879915714263916 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.880152702331543 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.879900932312012 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.880235195159912 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.880332946777344 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.88004732131958 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.880288124084473 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.880094051361084 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.880321979522705 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.879859924316406 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.880184650421143 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.880111217498779 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.88018274307251 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.880171775817871 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.8800811767578125 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.879889488220215 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.880235195159912 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.880288124084473 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.880133628845215 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.8803558349609375 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.880610942840576 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.879991054534912 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.880197525024414 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.879972457885742 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.880382061004639 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.880382061004639 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.880282402038574 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.88029146194458 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.880246639251709 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.879754066467285 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.880250453948975 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.880026817321777 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.880145072937012 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.880167484283447 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.880327224731445 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.88020133972168 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.880143642425537 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.879705429077148 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.880107879638672 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.8800811767578125 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.88020133972168 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.880127906799316 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.880321979522705 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.880099773406982 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.88029146194458 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.880364894866943 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.880343914031982 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.879992961883545 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.8800811767578125 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.880017280578613 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.880258083343506 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.880012035369873 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.8802714347839355 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.880458831787109 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.880407810211182 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.880453586578369 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.879848480224609 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.880195617675781 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.879605770111084 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.880407810211182 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.880332946777344 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.880640983581543 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.880573272705078 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.880295276641846 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.8800249099731445 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.8803558349609375 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.880305290222168 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.8803839683532715 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.879970550537109 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.88018274307251 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.880083084106445 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.880154609680176 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.879299640655518 |
| } |
| ] |
| }, |
| { |
| "bit": 15, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.851803779602051 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.851534843444824 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.851141452789307 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.851541996002197 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.852069854736328 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.851837635040283 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.851553440093994 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.85181188583374 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.851686477661133 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.8524909019470215 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.851568222045898 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.85197639465332 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.852665424346924 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.851968765258789 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.851800918579102 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.85217809677124 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.851399898529053 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.85080623626709 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.852111339569092 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.851820945739746 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.853028297424316 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.851682662963867 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.851198196411133 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.850396633148193 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.851465225219727 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.851661682128906 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.851912975311279 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.851450443267822 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.851893901824951 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.851579189300537 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.850673675537109 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.851841449737549 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.851587295532227 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.852566242218018 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.852047443389893 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.851577281951904 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.852357864379883 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.852042198181152 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.8522047996521 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.851201057434082 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.850250720977783 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.850669860839844 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.85175895690918 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.851264953613281 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.852581024169922 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.85195779800415 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.851490020751953 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.851355075836182 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.852105617523193 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.851989269256592 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.851344108581543 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.852625846862793 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.852175235748291 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.851635932922363 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.851373672485352 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.851502895355225 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.8521223068237305 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.851395606994629 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.85249662399292 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.852126121520996 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.8512725830078125 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.8511433601379395 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.851635932922363 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.851790904998779 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.85185432434082 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.851107597351074 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.851478099822998 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.851834297180176 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.851776123046875 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.851751804351807 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.851510047912598 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.851768970489502 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.851160526275635 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.85174036026001 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.851614952087402 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.85175895690918 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.851901054382324 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.851712226867676 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.851632118225098 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.85183048248291 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.851901054382324 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.851733207702637 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.851454257965088 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.851933002471924 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.851674556732178 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.851592540740967 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.851182460784912 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.850868225097656 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.851665496826172 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.851873397827148 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.85175895690918 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.851669788360596 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.8516845703125 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.851369857788086 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.851444244384766 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.851452350616455 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.851781845092773 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.850999355316162 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.851558685302734 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.851351261138916 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.851418495178223 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.851422309875488 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.851661682128906 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.851663589477539 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.851762771606445 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.851757049560547 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.851639270782471 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.851624011993408 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.8516526222229 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.851980209350586 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.851414680480957 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.851186275482178 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.85186767578125 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.851674556732178 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.851508140563965 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.851630210876465 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.851328372955322 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.851654529571533 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.851721286773682 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.851672649383545 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.851824760437012 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.8512349128723145 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.851358890533447 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.851388931274414 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.851497173309326 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.851109504699707 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.851579189300537 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.851737022399902 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.851966857910156 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.851499080657959 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.851687908172607 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.851787090301514 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.8519744873046875 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.851686477661133 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.851699352264404 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.851418495178223 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.851437568664551 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.851767063140869 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.851837635040283 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.851572513580322 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.851714134216309 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.851837635040283 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.85183572769165 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.851737022399902 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.851938724517822 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.851998329162598 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.851687908172607 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.851570129394531 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.851674556732178 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.851616859436035 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.851815700531006 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.851566314697266 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.85153865814209 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.851866245269775 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.851650714874268 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.851646900177002 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.851729393005371 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.851805686950684 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.851399898529053 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.851564407348633 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.851678848266602 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.851800918579102 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.851672649383545 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.851480007171631 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.851749897003174 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.851306438446045 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.852004528045654 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.851478099822998 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.851747989654541 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.85174036026001 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.8517351150512695 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.851785659790039 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.851470947265625 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.851229667663574 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.851358890533447 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.851624011993408 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.851800918579102 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.851800918579102 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.85175895690918 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.851785659790039 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.852242469787598 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.8516526222229 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.85155725479126 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.85175895690918 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.851934909820557 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.851843357086182 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.8514275550842285 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.851532936096191 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.852152347564697 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.851849555969238 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.851785659790039 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.851898193359375 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.851841449737549 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.85195779800415 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.8519182205200195 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.851534843444824 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.851659774780273 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.8517045974731445 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.85186243057251 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.8515944480896 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.851792812347412 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.851499080657959 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.8518829345703125 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.851834297180176 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.851663589477539 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.851680755615234 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.851504802703857 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.851819038391113 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.851916790008545 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.851949691772461 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.851828575134277 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.851654529571533 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.851800918579102 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.851768970489502 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.851674556732178 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.85249662399292 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.851317405700684 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.851624011993408 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.851463317871094 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.8518452644348145 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.851373672485352 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.851847171783447 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.850452423095703 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.8510332107543945 |
| } |
| ] |
| } |
| ] |
| } |
