Public Shared Function AvgPool2d(x As Tensor,kernelH As Integer,kernelW As Integer,strideY As Integer,strideX As Integer,paddingY As Integer,paddingX As Integer,dilationY As Integer,dilationX As Integer) As TensorDim x_width As Integer = Marshal.ReadInt32(IntPtr.Add(x.Shape, 2 * Marshal.SizeOf(GetType(Integer))))Dim x_height As Integer = Marshal.ReadInt32(IntPtr.Add(x.Shape, Marshal.SizeOf(GetType(Integer))))Dim x_channel As Integer = Marshal.ReadInt32(x.Shape)Dim y_width As Integer = (x_width + 2 * paddingX - dilationX * (kernelW - 1) - 1) \ strideX + 1Dim y_height As Integer = (x_height + 2 * paddingY - dilationY * (kernelH - 1) - 1) \ strideY + 1Dim y_channel As Integer = x_channelDim y As New Tensor(y_channel, y_height, y_width)Dim winsize As Single = CSng(kernelW * kernelH)Parallel.For(0, x_channel, Sub(c As Integer)For ox As Integer = 0 To y_width - 1Dim ix_ As Integer = ox * strideX - paddingXFor oy As Integer = 0 To y_height - 1Dim iy_ As Integer = oy * strideY - paddingYDim mean As Single = 0.0FFor fx As Integer = 0 To kernelW - 1Dim ix As Integer = ix_ + fx * dilationXIf ix >= x_width OrElse ix < 0 ThenContinue ForEnd IfFor fy As Integer = 0 To kernelH - 1Dim iy As Integer = iy_ + fy * dilationYIf iy >= x_height OrElse iy < 0 ThenContinue ForEnd IfDim offset As Integer = ((c * x_height + iy) * x_width + ix) * Marshal.SizeOf(GetType(Single))mean += Marshal.PtrToStructure(Of Single)(IntPtr.Add(x.Data, offset))NextNextDim resultOffset As Integer = ((c * y_height + oy) * y_width + ox) * Marshal.SizeOf(GetType(Single))Marshal.StructureToPtr(mean / winsize, IntPtr.Add(y.Data, resultOffset), False)NextNextEnd Sub)Return yEnd FunctionPublic Shared Function BatchNorm2d_(x As Tensor, mean As Tensor, [var] As Tensor, weight As Tensor, bias As Tensor) As TensorDim hw As Integer = Marshal.ReadInt32(IntPtr.Add(x.Shape, Marshal.SizeOf(GetType(Integer)))) *Marshal.ReadInt32(IntPtr.Add(x.Shape, 2 * Marshal.SizeOf(GetType(Integer))))Dim pmean As IntPtr = mean.DataDim pvar As IntPtr = [var].DataDim pw As IntPtr = weight.DataDim pb As IntPtr = bias.DataDim px As IntPtr = x.DataDim endPtr As IntPtr = IntPtr.Add(x.Data, x.Numel * Marshal.SizeOf(GetType(Single)))While px.ToInt64() < endPtr.ToInt64()Dim currentMean As Single = Marshal.PtrToStructure(Of Single)(pmean)Dim currentVar As Single = Marshal.PtrToStructure(Of Single)(pvar)Dim currentWeight As Single = Marshal.PtrToStructure(Of Single)(pw)Dim currentBias As Single = Marshal.PtrToStructure(Of Single)(pb)For i As Integer = 0 To hw - 1Dim offset As Integer = i * Marshal.SizeOf(GetType(Single))Dim value As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(px, offset))value = (value - currentMean) / currentVar * currentWeight + currentBiasMarshal.StructureToPtr(value, IntPtr.Add(px, offset), False)Nextpx = IntPtr.Add(px, hw * Marshal.SizeOf(GetType(Single)))pmean = IntPtr.Add(pmean, Marshal.SizeOf(GetType(Single)))pvar = IntPtr.Add(pvar, Marshal.SizeOf(GetType(Single)))pw = IntPtr.Add(pw, Marshal.SizeOf(GetType(Single)))pb = IntPtr.Add(pb, Marshal.SizeOf(GetType(Single)))End WhileReturn xEnd FunctionPublic Shared Function Conv2d(x As Tensor,weight As Tensor,bias As Tensor,padY As Integer,padX As Integer,padH As Integer,padW As Integer,strideY As Integer,strideX As Integer,dilationY As Integer,dilationX As Integer,group As Integer) As TensorDim srcC As Integer = Marshal.ReadInt32(x.Shape)Dim srcH As Integer = Marshal.ReadInt32(IntPtr.Add(x.Shape, Marshal.SizeOf(GetType(Integer))))Dim srcW As Integer = Marshal.ReadInt32(IntPtr.Add(x.Shape, 2 * Marshal.SizeOf(GetType(Integer))))Dim kernelY As Integer = Marshal.ReadInt32(IntPtr.Add(weight.Shape, 2 * Marshal.SizeOf(GetType(Integer))))Dim kernelX As Integer = Marshal.ReadInt32(IntPtr.Add(weight.Shape, 3 * Marshal.SizeOf(GetType(Integer))))Dim dstC As Integer = Marshal.ReadInt32(weight.Shape)Dim dstH As Integer = (srcH + padY + padH - (dilationY * (kernelY - 1) + 1)) \ strideY + 1Dim dstW As Integer = (srcW + padX + padW - (dilationX * (kernelX - 1) + 1)) \ strideX + 1Dim y As New Tensor(dstC, dstH, dstW)Dim xBuffer(srcC * srcH * srcW - 1) As SingleMarshal.Copy(x.Data, xBuffer, 0, xBuffer.Length)Dim weightBuffer(weight.Numel - 1) As SingleMarshal.Copy(weight.Data, weightBuffer, 0, weightBuffer.Length)Dim biasBuffer As Single() = NothingIf bias IsNot Nothing ThenIf bias.Data = IntPtr.Zero Then Throw New ArgumentException("bias tensor data is invalid")biasBuffer = New Single(bias.Numel - 1) {}Marshal.Copy(bias.Data, biasBuffer, 0, biasBuffer.Length)End IfDim resultBuffer(dstC * dstH * dstW - 1) As SingleDim dstC_per_group As Integer = dstC \ groupDim srcC_per_group As Integer = srcC \ groupDim weight_base As Integer = srcC_per_group * kernelY * kernelXParallel.For(0, group, Sub(g)Dim curGroup = gDim src_base1 As Integer = g * srcC_per_groupDim dst_base1 As Integer = g * dstC_per_groupDim groupBias As Single() = NothingIf bias IsNot Nothing ThengroupBias = New Single(dstC_per_group - 1) {}Array.Copy(biasBuffer, g * dstC_per_group, groupBias, 0, dstC_per_group)End IfParallel.For(0, dstH, Sub(dy)Dim buffer(srcC_per_group * kernelY * kernelX - 1) As SingleDim sy1 As Integer = dy * strideY - padYFor dx As Integer = 0 To dstW - 1Dim sx1 As Integer = dx * strideX - padXDim bufferIdx As Integer = 0For sc As Integer = 0 To srcC_per_group - 1Dim src_base2 As Integer = (src_base1 + sc) * srcHFor ky As Integer = 0 To kernelY - 1Dim sy As Integer = sy1 + ky * dilationYIf sy < 0 OrElse sy >= srcH ThenFor kx As Integer = 0 To kernelX - 1buffer(bufferIdx) = 0.0FbufferIdx += 1NextElseDim src_offset As Integer = (src_base2 + sy) * srcWFor kx As Integer = 0 To kernelX - 1Dim sx As Integer = sx1 + kx * dilationXIf sx >= 0 AndAlso sx < srcW Thenbuffer(bufferIdx) = xBuffer(src_offset + sx)Elsebuffer(bufferIdx) = 0.0FEnd IfbufferIdx += 1NextEnd IfNextNextFor dc As Integer = 0 To dstC_per_group - 1Dim sum As Single = 0.0FDim weight_offset As Integer = (curGroup * dstC_per_group + dc) * weight_baseFor m As Integer = 0 To weight_base - 1sum += buffer(m) * weightBuffer(weight_offset + m)NextIf bias IsNot Nothing Thensum += groupBias(dc)End IfDim out_idx As Integer = ((dst_base1 + dc) * dstH + dy) * dstW + dxresultBuffer(out_idx) = sumNextNextEnd Sub)End Sub)Marshal.Copy(resultBuffer, 0, y.Data, resultBuffer.Length)Return yEnd FunctionPublic Shared Function EltwiseMulScalar_(x As Tensor, s As Single) As TensorDim ptr As IntPtr = x.DataFor i As Integer = 0 To x.Numel - 1Dim value As Single = Marshal.PtrToStructure(Of Single)(ptr)value *= sMarshal.StructureToPtr(value, ptr, False)ptr = IntPtr.Add(ptr, Marshal.SizeOf(GetType(Single)))NextReturn xEnd FunctionPublic Shared Function MatMul(a As Tensor, b As Tensor) As TensorDim aw As Integer = Marshal.ReadInt32(IntPtr.Add(a.Shape, Marshal.SizeOf(GetType(Integer))))Dim ah As Integer = Marshal.ReadInt32(a.Shape)Dim bw As Integer = Marshal.ReadInt32(IntPtr.Add(b.Shape, Marshal.SizeOf(GetType(Integer))))Dim bh As Integer = Marshal.ReadInt32(b.Shape)b = b.Transpose2d()Dim c As New Tensor(ah, bw)Dim vectorSize As Integer = Vector(Of Single).CountDim vectorCount As Integer = bh \ vectorSizeParallel.For(0, ah, Sub(i)For j As Integer = 0 To bw - 1Dim sumVec As Vector(Of Single) = Vector(Of Single).ZeroFor k As Integer = 0 To vectorCount - 1Dim offset As Integer = k * vectorSizeDim aVec As Vector(Of Single) = a.GetVector(i * aw + offset)Dim bVec As Vector(Of Single) = b.GetVector(j * bh + offset)sumVec += aVec * bVecNextDim sum As Single = 0For k As Integer = 0 To vectorSize - 1sum += sumVec(k)NextFor k As Integer = vectorCount * vectorSize To bh - 1sum += a(i * aw + k) * b(j * bh + k)Nextc(i * bw + j) = sumNextEnd Sub)Return cEnd FunctionPublic Shared Function MaxPool2d(x As Tensor,kernelH As Integer,kernelW As Integer,strideY As Integer,strideX As Integer,paddingY As Integer,paddingX As Integer,dilationY As Integer,dilationX As Integer) As TensorDim x_width As Integer = Marshal.ReadInt32(IntPtr.Add(x.Shape, 2 * Marshal.SizeOf(GetType(Integer))))Dim x_height As Integer = Marshal.ReadInt32(IntPtr.Add(x.Shape, Marshal.SizeOf(GetType(Integer))))Dim x_channel As Integer = Marshal.ReadInt32(x.Shape)Dim y_width As Integer = (x_width + 2 * paddingX - dilationX * (kernelW - 1) - 1) \ strideX + 1Dim y_height As Integer = (x_height + 2 * paddingY - dilationY * (kernelH - 1) - 1) \ strideY + 1Dim y_channel As Integer = x_channelDim y As New Tensor(y_channel, y_height, y_width)Parallel.For(0, x_channel, Sub(c As Integer)For ox As Integer = 0 To y_width - 1Dim ix_ As Integer = ox * strideX - paddingXFor oy As Integer = 0 To y_height - 1Dim iy_ As Integer = oy * strideY - paddingYDim max As Single = Single.MinValueFor fx As Integer = 0 To kernelW - 1Dim ix As Integer = ix_ + fx * dilationXIf ix >= x_width OrElse ix < 0 ThenContinue ForEnd IfFor fy As Integer = 0 To kernelH - 1Dim iy As Integer = iy_ + fy * dilationYIf iy >= x_height OrElse iy < 0 ThenContinue ForEnd IfDim offset As Integer = ((c * x_height + iy) * x_width + ix) * Marshal.SizeOf(GetType(Single))Dim v As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(x.Data, offset))If v > max Thenmax = vEnd IfNextNextDim resultOffset As Integer = ((c * y_height + oy) * y_width + ox) * Marshal.SizeOf(GetType(Single))Marshal.StructureToPtr(max, IntPtr.Add(y.Data, resultOffset), False)NextNextEnd Sub)Return yEnd FunctionPublic Shared Function PixelShuffle(x As Tensor) As TensorDim x_depth As Integer = Marshal.ReadInt32(x.Shape)Dim x_height As Integer = Marshal.ReadInt32(IntPtr.Add(x.Shape, Marshal.SizeOf(GetType(Integer))))Dim x_width As Integer = Marshal.ReadInt32(IntPtr.Add(x.Shape, 2 * Marshal.SizeOf(GetType(Integer))))Dim y As New Tensor(x_depth \ 4, x_height * 2, x_width * 2)Dim y_depth As Integer = Marshal.ReadInt32(y.Shape)Dim y_height As Integer = Marshal.ReadInt32(IntPtr.Add(y.Shape, Marshal.SizeOf(GetType(Integer))))Dim y_width As Integer = Marshal.ReadInt32(IntPtr.Add(y.Shape, 2 * Marshal.SizeOf(GetType(Integer))))For od As Integer = 0 To y_depth - 1Dim id As Integer = od * 4For oy As Integer = 0 To y_height - 1 Step 2Dim iy As Integer = oy \ 2For ox As Integer = 0 To y_width - 1 Step 2Dim ix As Integer = ox \ 2Dim val1 As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(x.Data, ((id * x_height + iy) * x_width + ix) * Marshal.SizeOf(GetType(Single))))Dim val2 As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(x.Data, (((id + 1) * x_height + iy) * x_width + ix) * Marshal.SizeOf(GetType(Single))))Dim val3 As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(x.Data, (((id + 2) * x_height + iy) * x_width + ix) * Marshal.SizeOf(GetType(Single))))Dim val4 As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(x.Data, (((id + 3) * x_height + iy) * x_width + ix) * Marshal.SizeOf(GetType(Single))))Marshal.StructureToPtr(val1, IntPtr.Add(y.Data, ((od * y_height + oy) * y_width + ox) * Marshal.SizeOf(GetType(Single))), False)Marshal.StructureToPtr(val2, IntPtr.Add(y.Data, ((od * y_height + oy) * y_width + ox + 1) * Marshal.SizeOf(GetType(Single))), False)Marshal.StructureToPtr(val3, IntPtr.Add(y.Data, ((od * y_height + oy + 1) * y_width + ox) * Marshal.SizeOf(GetType(Single))), False)Marshal.StructureToPtr(val4, IntPtr.Add(y.Data, ((od * y_height + oy + 1) * y_width + ox + 1) * Marshal.SizeOf(GetType(Single))), False)NextNextNextReturn yEnd FunctionPublic Shared Function Plus_(a As Tensor, b As Tensor) As TensorDim vectorSize As Integer = Vector(Of Single).CountDim vectorCount As Integer = a.Numel \ vectorSizeFor i As Integer = 0 To vectorCount - 1Dim offset As Integer = i * vectorSizeDim aVec As Vector(Of Single) = a.GetVector(offset)Dim bVec As Vector(Of Single) = b.GetVector(offset)a.SetVector(offset, aVec + bVec)NextFor i As Integer = vectorCount * vectorSize To a.Numel - 1a(i) += b(i)NextReturn aEnd FunctionPublic Shared Function RestrictedCat2d(a As Tensor, b As Tensor) As TensorDim height As Integer = Math.Min(Marshal.ReadInt32(IntPtr.Add(a.Shape, Marshal.SizeOf(GetType(Integer)))),Marshal.ReadInt32(IntPtr.Add(b.Shape, Marshal.SizeOf(GetType(Integer)))))Dim width As Integer = Math.Min(Marshal.ReadInt32(IntPtr.Add(a.Shape, 2 * Marshal.SizeOf(GetType(Integer)))),Marshal.ReadInt32(IntPtr.Add(b.Shape, 2 * Marshal.SizeOf(GetType(Integer)))))Dim adepth As Integer = Marshal.ReadInt32(a.Shape)Dim bdepth As Integer = Marshal.ReadInt32(b.Shape)Dim aheight As Integer = Marshal.ReadInt32(IntPtr.Add(a.Shape, Marshal.SizeOf(GetType(Integer))))Dim bheight As Integer = Marshal.ReadInt32(IntPtr.Add(b.Shape, Marshal.SizeOf(GetType(Integer))))Dim awidth As Integer = Marshal.ReadInt32(IntPtr.Add(a.Shape, 2 * Marshal.SizeOf(GetType(Integer))))Dim bwidth As Integer = Marshal.ReadInt32(IntPtr.Add(b.Shape, 2 * Marshal.SizeOf(GetType(Integer))))Dim depth As Integer = adepth + bdepthDim c As New Tensor(depth, height, width)For y As Integer = 0 To height - 1For x As Integer = 0 To width - 1Dim gd As Integer = 0For d As Integer = 0 To adepth - 1Dim a_val As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(a.Data, ((d * aheight + y) * awidth + x) * Marshal.SizeOf(GetType(Single))))Marshal.StructureToPtr(a_val, IntPtr.Add(c.Data, ((gd * height + y) * width + x) * Marshal.SizeOf(GetType(Single))), False)gd += 1NextFor d As Integer = 0 To bdepth - 1Dim b_val As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(b.Data, ((d * bheight + y) * bwidth + x) * Marshal.SizeOf(GetType(Single))))Marshal.StructureToPtr(b_val, IntPtr.Add(c.Data, ((gd * height + y) * width + x) * Marshal.SizeOf(GetType(Single))), False)gd += 1NextNextNextReturn cEnd FunctionPublic Shared Function Sigmoid_(x As Tensor) As TensorDim px As IntPtr = x.DataFor i As Integer = 0 To x.Numel - 1Dim value As Single = Marshal.PtrToStructure(Of Single)(px)value = 1.0F / (1.0F + CSng(Math.Exp(-value)))Marshal.StructureToPtr(value, px, False)px = IntPtr.Add(px, Marshal.SizeOf(GetType(Single)))NextReturn xEnd FunctionPublic Shared Function Softmax2d(input As Tensor) As TensorDim height As Integer = Marshal.ReadInt32(input.Shape)Dim width As Integer = Marshal.ReadInt32(IntPtr.Add(input.Shape, Marshal.SizeOf(GetType(Integer))))Dim result As New Tensor(height, width)For y As Integer = 0 To height - 1Dim amax As Single = Single.MinValueFor x As Integer = 0 To width - 1Dim v As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(input.Data, (y * width + x) * Marshal.SizeOf(GetType(Single))))If amax < v Thenamax = vEnd IfNextDim sum As Single = 0.0FFor x As Integer = 0 To width - 1Dim v As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(input.Data, (y * width + x) * Marshal.SizeOf(GetType(Single))))sum += CSng(Math.Exp(v - amax))NextFor x As Integer = 0 To width - 1Dim v As Single = Marshal.PtrToStructure(Of Single)(IntPtr.Add(input.Data, (y * width + x) * Marshal.SizeOf(GetType(Single))))Dim resultVal As Single = CSng(Math.Exp(v - amax) / sum)Marshal.StructureToPtr(resultVal, IntPtr.Add(result.Data, (y * width + x) * Marshal.SizeOf(GetType(Single))), False)NextNextReturn resultEnd Function