1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 /** 18 * Tests for halving-add idiomatic vectorization. 19 * 20 * Alternative version expressed with logical shift right 21 * in the higher precision (has no impact on idiom). 22 */ 23 public class HaddAltShort { 24 25 private static final int N = 64 * 1024; 26 private static final int M = N + 31; 27 28 static short[] sB1 = new short[M]; 29 static short[] sB2 = new short[M]; 30 static short[] sBo = new short[M]; 31 32 /// CHECK-START: void HaddAltShort.halving_add_signed(short[], short[], short[]) loop_optimization (before) 33 /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none 34 /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none 35 /// CHECK-DAG: <<Get1:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 36 /// CHECK-DAG: <<Get2:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 37 /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none 38 /// CHECK-DAG: <<UShr:i\d+>> UShr [<<Add>>,<<I1>>] loop:<<Loop>> outer_loop:none 39 /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<UShr>>] loop:<<Loop>> outer_loop:none 40 /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none 41 // 42 /// CHECK-START-{ARM,ARM64}: void HaddAltShort.halving_add_signed(short[], short[], short[]) loop_optimization (after) 43 /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none 44 /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none 45 /// CHECK-DAG: <<HAdd:d\d+>> VecHalvingAdd [<<Get1>>,<<Get2>>] packed_type:Int16 rounded:false loop:<<Loop>> outer_loop:none 46 /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<HAdd>>] loop:<<Loop>> outer_loop:none halving_add_signed(short[] b1, short[] b2, short[] bo)47 private static void halving_add_signed(short[] b1, short[] b2, short[] bo) { 48 int min_length = Math.min(bo.length, Math.min(b1.length, b2.length)); 49 for (int i = 0; i < min_length; i++) { 50 bo[i] = (short) ((b1[i] + b2[i]) >>> 1); 51 } 52 } 53 54 /// CHECK-START: void HaddAltShort.halving_add_unsigned(short[], short[], short[]) instruction_simplifier (before) 55 /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none 56 /// CHECK-DAG: <<UMAX:i\d+>> IntConstant 65535 loop:none 57 /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none 58 /// CHECK-DAG: <<Get1:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 59 /// CHECK-DAG: <<Get2:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 60 /// CHECK-DAG: <<And1:i\d+>> And [<<Get1>>,<<UMAX>>] loop:<<Loop>> outer_loop:none 61 /// CHECK-DAG: <<And2:i\d+>> And [<<UMAX>>,<<Get2>>] loop:<<Loop>> outer_loop:none 62 /// CHECK-DAG: <<Add:i\d+>> Add [<<And1>>,<<And2>>] loop:<<Loop>> outer_loop:none 63 /// CHECK-DAG: <<UShr:i\d+>> UShr [<<Add>>,<<I1>>] loop:<<Loop>> outer_loop:none 64 /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<UShr>>] loop:<<Loop>> outer_loop:none 65 /// CHECK-DAG: ArraySet [{{l\d+}},{{i\d+}},<<Cnv>>] loop:<<Loop>> outer_loop:none 66 // 67 /// CHECK-START: void HaddAltShort.halving_add_unsigned(short[], short[], short[]) loop_optimization (before) 68 /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none 69 /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none 70 /// CHECK-DAG: <<Get1:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none 71 /// CHECK-DAG: <<Get2:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none 72 /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none 73 /// CHECK-DAG: <<UShr:i\d+>> UShr [<<Add>>,<<I1>>] loop:<<Loop>> outer_loop:none 74 /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<UShr>>] loop:<<Loop>> outer_loop:none 75 /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none 76 // 77 /// CHECK-START-{ARM,ARM64}: void HaddAltShort.halving_add_unsigned(short[], short[], short[]) loop_optimization (after) 78 /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none 79 /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none 80 /// CHECK-DAG: <<HAdd:d\d+>> VecHalvingAdd [<<Get1>>,<<Get2>>] packed_type:Uint16 rounded:false loop:<<Loop>> outer_loop:none 81 /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<HAdd>>] loop:<<Loop>> outer_loop:none halving_add_unsigned(short[] b1, short[] b2, short[] bo)82 private static void halving_add_unsigned(short[] b1, short[] b2, short[] bo) { 83 int min_length = Math.min(bo.length, Math.min(b1.length, b2.length)); 84 for (int i = 0; i < min_length; i++) { 85 int v1 = b1[i] & 0xffff; 86 int v2 = b2[i] & 0xffff; 87 bo[i] = (short) ((v1 + v2) >>> 1); 88 } 89 } 90 91 /// CHECK-START: void HaddAltShort.rounding_halving_add_signed(short[], short[], short[]) loop_optimization (before) 92 /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none 93 /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none 94 /// CHECK-DAG: <<Get1:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 95 /// CHECK-DAG: <<Get2:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 96 /// CHECK-DAG: <<Add1:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none 97 /// CHECK-DAG: <<Add2:i\d+>> Add [<<Add1>>,<<I1>>] loop:<<Loop>> outer_loop:none 98 /// CHECK-DAG: <<UShr:i\d+>> UShr [<<Add2>>,<<I1>>] loop:<<Loop>> outer_loop:none 99 /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<UShr>>] loop:<<Loop>> outer_loop:none 100 /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none 101 // 102 /// CHECK-START-{ARM,ARM64}: void HaddAltShort.rounding_halving_add_signed(short[], short[], short[]) loop_optimization (after) 103 /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none 104 /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none 105 /// CHECK-DAG: <<HAdd:d\d+>> VecHalvingAdd [<<Get1>>,<<Get2>>] packed_type:Int16 rounded:true loop:<<Loop>> outer_loop:none 106 /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<HAdd>>] loop:<<Loop>> outer_loop:none rounding_halving_add_signed(short[] b1, short[] b2, short[] bo)107 private static void rounding_halving_add_signed(short[] b1, short[] b2, short[] bo) { 108 int min_length = Math.min(bo.length, Math.min(b1.length, b2.length)); 109 for (int i = 0; i < min_length; i++) { 110 bo[i] = (short) ((b1[i] + b2[i] + 1) >>> 1); 111 } 112 } 113 114 /// CHECK-START: void HaddAltShort.rounding_halving_add_unsigned(short[], short[], short[]) instruction_simplifier (before) 115 /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none 116 /// CHECK-DAG: <<UMAX:i\d+>> IntConstant 65535 loop:none 117 /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none 118 /// CHECK-DAG: <<Get1:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 119 /// CHECK-DAG: <<Get2:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 120 /// CHECK-DAG: <<And1:i\d+>> And [<<Get1>>,<<UMAX>>] loop:<<Loop>> outer_loop:none 121 /// CHECK-DAG: <<And2:i\d+>> And [<<UMAX>>,<<Get2>>] loop:<<Loop>> outer_loop:none 122 /// CHECK-DAG: <<Add1:i\d+>> Add [<<And1>>,<<And2>>] loop:<<Loop>> outer_loop:none 123 /// CHECK-DAG: <<Add2:i\d+>> Add [<<Add1>>,<<I1>>] loop:<<Loop>> outer_loop:none 124 /// CHECK-DAG: <<UShr:i\d+>> UShr [<<Add2>>,<<I1>>] loop:<<Loop>> outer_loop:none 125 /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<UShr>>] loop:<<Loop>> outer_loop:none 126 /// CHECK-DAG: ArraySet [{{l\d+}},{{i\d+}},<<Cnv>>] loop:<<Loop>> outer_loop:none 127 // 128 /// CHECK-START: void HaddAltShort.rounding_halving_add_unsigned(short[], short[], short[]) loop_optimization (before) 129 /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none 130 /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none 131 /// CHECK-DAG: <<Get1:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none 132 /// CHECK-DAG: <<Get2:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none 133 /// CHECK-DAG: <<Add1:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none 134 /// CHECK-DAG: <<Add2:i\d+>> Add [<<Add1>>,<<I1>>] loop:<<Loop>> outer_loop:none 135 /// CHECK-DAG: <<UShr:i\d+>> UShr [<<Add2>>,<<I1>>] loop:<<Loop>> outer_loop:none 136 /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<UShr>>] loop:<<Loop>> outer_loop:none 137 /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none 138 // 139 /// CHECK-START-{ARM,ARM64}: void HaddAltShort.rounding_halving_add_unsigned(short[], short[], short[]) loop_optimization (after) 140 /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none 141 /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none 142 /// CHECK-DAG: <<HAdd:d\d+>> VecHalvingAdd [<<Get1>>,<<Get2>>] packed_type:Uint16 rounded:true loop:<<Loop>> outer_loop:none 143 /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<HAdd>>] loop:<<Loop>> outer_loop:none rounding_halving_add_unsigned(short[] b1, short[] b2, short[] bo)144 private static void rounding_halving_add_unsigned(short[] b1, short[] b2, short[] bo) { 145 int min_length = Math.min(bo.length, Math.min(b1.length, b2.length)); 146 for (int i = 0; i < min_length; i++) { 147 int v1 = b1[i] & 0xffff; 148 int v2 = b2[i] & 0xffff; 149 bo[i] = (short) ((v1 + v2 + 1) >>> 1); 150 } 151 } 152 153 /// CHECK-START: void HaddAltShort.halving_add_signed_constant(short[], short[]) loop_optimization (before) 154 /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none 155 /// CHECK-DAG: <<SMAX:i\d+>> IntConstant 32767 loop:none 156 /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none 157 /// CHECK-DAG: <<Get:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 158 /// CHECK-DAG: <<Add:i\d+>> Add [<<Get>>,<<SMAX>>] loop:<<Loop>> outer_loop:none 159 /// CHECK-DAG: <<UShr:i\d+>> UShr [<<Add>>,<<I1>>] loop:<<Loop>> outer_loop:none 160 /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<UShr>>] loop:<<Loop>> outer_loop:none 161 /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none 162 // 163 /// CHECK-START-{ARM,ARM64}: void HaddAltShort.halving_add_signed_constant(short[], short[]) loop_optimization (after) 164 /// CHECK-DAG: <<SMAX:i\d+>> IntConstant 32767 loop:none 165 /// CHECK-DAG: <<Repl:d\d+>> VecReplicateScalar [<<SMAX>>] loop:none 166 /// CHECK-DAG: <<Get:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none 167 /// CHECK-DAG: <<HAdd:d\d+>> VecHalvingAdd [<<Get>>,<<Repl>>] packed_type:Int16 rounded:false loop:<<Loop>> outer_loop:none 168 /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<HAdd>>] loop:<<Loop>> outer_loop:none halving_add_signed_constant(short[] b1, short[] bo)169 private static void halving_add_signed_constant(short[] b1, short[] bo) { 170 int min_length = Math.min(bo.length, b1.length); 171 for (int i = 0; i < min_length; i++) { 172 bo[i] = (short) ((b1[i] + 0x7fff) >>> 1); 173 } 174 } 175 176 /// CHECK-START: void HaddAltShort.halving_add_unsigned_constant(short[], short[]) instruction_simplifier (before) 177 /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none 178 /// CHECK-DAG: <<UMAX:i\d+>> IntConstant 65535 loop:none 179 /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none 180 /// CHECK-DAG: <<Get:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none 181 /// CHECK-DAG: <<And:i\d+>> And [<<Get>>,<<UMAX>>] loop:<<Loop>> outer_loop:none 182 /// CHECK-DAG: <<Add:i\d+>> Add [<<And>>,<<UMAX>>] loop:<<Loop>> outer_loop:none 183 /// CHECK-DAG: <<UShr:i\d+>> UShr [<<Add>>,<<I1>>] loop:<<Loop>> outer_loop:none 184 /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<UShr>>] loop:<<Loop>> outer_loop:none 185 /// CHECK-DAG: ArraySet [{{l\d+}},{{i\d+}},<<Cnv>>] loop:<<Loop>> outer_loop:none 186 // 187 /// CHECK-START: void HaddAltShort.halving_add_unsigned_constant(short[], short[]) loop_optimization (before) 188 /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none 189 /// CHECK-DAG: <<UMAX:i\d+>> IntConstant 65535 loop:none 190 /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none 191 /// CHECK-DAG: <<Get:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none 192 /// CHECK-DAG: <<Add:i\d+>> Add [<<Get>>,<<UMAX>>] loop:<<Loop>> outer_loop:none 193 /// CHECK-DAG: <<UShr:i\d+>> UShr [<<Add>>,<<I1>>] loop:<<Loop>> outer_loop:none 194 /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<UShr>>] loop:<<Loop>> outer_loop:none 195 /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none 196 // 197 /// CHECK-START-{ARM,ARM64}: void HaddAltShort.halving_add_unsigned_constant(short[], short[]) loop_optimization (after) 198 /// CHECK-DAG: <<UMAX:i\d+>> IntConstant 65535 loop:none 199 /// CHECK-DAG: <<Repl:d\d+>> VecReplicateScalar [<<UMAX>>] loop:none 200 /// CHECK-DAG: <<Get:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none 201 /// CHECK-DAG: <<HAdd:d\d+>> VecHalvingAdd [<<Get>>,<<Repl>>] packed_type:Uint16 rounded:false loop:<<Loop>> outer_loop:none 202 /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<HAdd>>] loop:<<Loop>> outer_loop:none halving_add_unsigned_constant(short[] b1, short[] bo)203 private static void halving_add_unsigned_constant(short[] b1, short[] bo) { 204 int min_length = Math.min(bo.length, b1.length); 205 for (int i = 0; i < min_length; i++) { 206 bo[i] = (short) (((b1[i] & 0xffff) + 0xffff) >>> 1); 207 } 208 } 209 main()210 public static void main() { 211 // Some interesting values. 212 short[] interesting = { 213 (short) 0x0000, 214 (short) 0x0001, 215 (short) 0x0002, 216 (short) 0x1234, 217 (short) 0x8000, 218 (short) 0x8001, 219 (short) 0x7fff, 220 (short) 0xffff 221 }; 222 // Initialize cross-values to test all cases, and also 223 // set up some extra values to exercise the cleanup loop. 224 for (int i = 0; i < M; i++) { 225 sB1[i] = (short) i; 226 sB2[i] = interesting[i & 7]; 227 } 228 229 // Test halving add idioms. Note that the expected result is computed 230 // with the arithmetic >> to demonstrate the computed narrower result 231 // does not depend on the wider >> or >>>. 232 halving_add_signed(sB1, sB2, sBo); 233 for (int i = 0; i < M; i++) { 234 short e = (short) ((sB1[i] + sB2[i]) >> 1); 235 expectEquals(e, sBo[i]); 236 } 237 halving_add_unsigned(sB1, sB2, sBo); 238 for (int i = 0; i < M; i++) { 239 short e = (short) (((sB1[i] & 0xffff) + (sB2[i] & 0xffff)) >> 1); 240 expectEquals(e, sBo[i]); 241 } 242 rounding_halving_add_signed(sB1, sB2, sBo); 243 for (int i = 0; i < M; i++) { 244 short e = (short) ((sB1[i] + sB2[i] + 1) >> 1); 245 expectEquals(e, sBo[i]); 246 } 247 rounding_halving_add_unsigned(sB1, sB2, sBo); 248 for (int i = 0; i < M; i++) { 249 short e = (short) (((sB1[i] & 0xffff) + (sB2[i] & 0xffff) + 1) >> 1); 250 expectEquals(e, sBo[i]); 251 } 252 halving_add_signed_constant(sB1, sBo); 253 for (int i = 0; i < M; i++) { 254 short e = (short) ((sB1[i] + 0x7fff) >> 1); 255 expectEquals(e, sBo[i]); 256 } 257 halving_add_unsigned_constant(sB1, sBo); 258 for (int i = 0; i < M; i++) { 259 short e = (short) (((sB1[i] & 0xffff) + 0xffff) >> 1); 260 expectEquals(e, sBo[i]); 261 } 262 263 System.out.println("HaddAltShort passed"); 264 } 265 expectEquals(int expected, int result)266 private static void expectEquals(int expected, int result) { 267 if (expected != result) { 268 throw new Error("Expected: " + expected + ", found: " + result); 269 } 270 } 271 } 272