From 52dc05f228df9c75ec6aa3ac44a634c6942c1f6a Mon Sep 17 00:00:00 2001 From: dzaima Date: Sat, 8 Apr 2023 20:04:23 +0300 Subject: [PATCH] SIMD +` --- src/builtins/scan.c | 44 +++++++----- src/singeli/src/scan.singeli | 129 +++++++++++++++++++++++++++++++---- 2 files changed, 144 insertions(+), 29 deletions(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index e6e6953c..f266d292 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -152,6 +152,32 @@ static B scan_lt(B x, u64 p, usz ia) { decG(x); return r; } +static B scan_plus(f64 r0, B x, u8 xe, usz ia) { + assert(xe!=el_bit && elNum(xe)); + B r; void* rp = m_tyarrv(&r, xe==el_f64? sizeof(f64) : sizeof(i32), ia, xe==el_f64? t_f64arr : t_i32arr); + #if SINGELI_AVX2 + switch(xe) { default:UD; + case el_i8: { if (!q_fi32(r0) || simd_scan_plus_i8_i32 (i8any_ptr(x), r0, rp, ia)!=ia) goto cs_i8_f64; decG(x); return r; } + case el_i16: { if (!q_fi32(r0) || simd_scan_plus_i16_i32(i16any_ptr(x), r0, rp, ia)!=ia) goto cs_i16_f64; decG(x); return r; } + case el_i32: { if (!q_fi32(r0) || simd_scan_plus_i32_i32(i32any_ptr(x), r0, rp, ia)!=ia) goto cs_i32_f64; decG(x); return r; } + case el_f64: { f64* xp=f64any_ptr(x); f64 c=r0; for (usz i=0; if; if (isAtm(x) || RNK(x)==0) thrM("`: Argument cannot have rank 0"); ur xr = RNK(x); @@ -169,13 +195,7 @@ B scan_c1(Md1D* d, B x) { B f = d->f; if (rtid==n_lt) return scan_lt(x, 0, ia); // < goto base; } - if (rtid==n_add) { // + - B r; void* rp = m_tyarrv(&r, xe==el_f64? sizeof(f64) : sizeof(i32), ia, xe==el_f64? t_f64arr : t_i32arr); - if (xe==el_i8 ) { i8* xp=i8any_ptr (x); i32 c=0; for (usz i=0; if; if (rtid==n_ceil ) return scan2_max_num(w, x, xe, ia); // ⌈ if (rtid==n_add) { // + - if (xe==el_f64) { f64 c=o2fG(w); f64* rp; B r=m_f64arrv(&rp, ia); f64* xp=f64any_ptr(x); for (usz i=0; if; return wv==0? t : C2(add, w, t); } - if (!q_i32(w) || !elInt(xe)) goto base; - i32 c = o2iG(w); - i32* rp; B r = m_i32arrv(&rp, ia); - if (xe==el_i8 ) { i8* xp=i8any_ptr (x); for (usz i=0; i { + def n = vcount{X} + def tb = width{E} * n + if (tb<=arch_defvw) tup{widen{[n]E, x}} + else if (1) { + assert{tb == 2*arch_defvw} + tup{ + widen{[n/2]E, half{x,0}}, + widen{[n/2]E, half{x,1}} + } + } + }, xs}} +} + +def floor{x & knum{x}} = x - (x%1) +def maxabsval{T & issigned{T}} = -minvalue{T} +def maxsafeint{T & issigned{T}} = maxvalue{T} +def maxsafeint{T==f64} = 1<<53 + +def simd_plus_scan{X, b, R}{x:*X, c:(R), r:*R, len:u64} = { + def bulk = arch_defvw/b + + def wd = (X!=R) & (width{X}<32) # whether to widen the working copy one size + def WE = tern{wd, ty_dbl{X}, X} # working copy element type + + # maxFastA: max absolute value for accumulator + # maxFastE: max absolute value for vector elements (not used if ~wd) + def maxFastE = if (wd) maxabsval{X} else maxabsval{X}/(bulk*tern{R==f64, 1, 4}) # 4 to give maxFastA some range + def maxFastA = maxsafeint{R} - maxFastE*bulk + + if (R!=f64) { def m = maxFastA + maxFastE*bulk; assert{m<=maxvalue{R}}; assert{-m>=minvalue{R}} } + + i:u64 = 0 + cv:= [arch_defvw/width{R}]R ** c + if (R==f64 and c != floor{c}) goto{'end'} + while (1) { + def ctmp = extract{cv,0} + if (max{ctmp,-ctmp} >= tern{R==f64, cast_i{f64, i64~~maxFastA}, maxFastA}) goto{'end'} + i2:= i+bulk + if (i2>len) goto{'end'} + def cx0 = tup{load{*[bulk]X~~(x+i)}} + def cx = if(wd) widenFull{WE,cx0} else cx0 + if (~wd) { # within-vector overflow check; widening gives range space for this to not happen + if (rare{homAny{tree_fold{|, each{{c:T} => absu{c} >= (ty_u{T}**maxFastE), cx}}}}) goto{'end'} + } + + def s0 = each{scan_plus, cx} + + def s1 = { + if (tuplen{s0}==1) s0 + else { def {v0,v1}=s0; tup{v0,v1+toLast{v0}} } + } + + def cr = eachx{+, widenFull{R, s1}, cv} + cv = toLast{tupsel{-1, cr}} + + each{{c:T} => assert{T==type{cv}}, cr} + assert{vcount{type{cv}} * tuplen{cr} == bulk} + + each{{c:T, j} => store{*T~~(r+i), j, c}, cr, iota{tuplen{cr}}} + i = i2 + } + setlabel{'end'} + + c = extract{cv, vcount{type{cv}}-1} + while (i < len) { + def {b,n} = addChk{c, promote{R,load{x,i}}} + if (rare{b}) return{i} + store{r, i, n} + c = n + ++i + } + len +} +fn simd_plus_scanG{X, b, R}(x:*X, c:R, r:*R, len:u64) : void = simd_plus_scan{X,b,R}{x, c, r, len} +fn simd_plus_scanC{X, b, R}(x:*X, c:R, r:*R, len:u64) : u64 = simd_plus_scan{X,b,R}{x, c, r, len} + +export{'simd_scan_plus_i8_i32', simd_plus_scanC{i8, 16, i32}} +export{'simd_scan_plus_i16_i32', simd_plus_scanC{i16, 16, i32}} +export{'simd_scan_plus_i32_i32', simd_plus_scanC{i32, 32, i32}} + +export{'simd_scan_plus_i16_f64', simd_plus_scanG{i16, 32, f64}} +export{'simd_scan_plus_i32_f64', simd_plus_scanG{i32, 32, f64}}