pytorch归一化

数据维度为[N,C,H,W]

1、BatchNorm

归一化维度:[N,H,W] 计算C次均值方差

import torch
from torch import nn


#BatchNorm2d
input = torch.randn((8,32,32,3))
input_ = input.permute(0,3,1,2)
print(input_.shape)

BN = nn.BatchNorm2d(3)
output = BN(input_)
print(output[:,0,:,:])

#test
X = input_[:,0,:,:]  #N,H,W
print(X.shape)

mean = torch.mean(X)
var = torch.var(X)
BN_one = (input_[:,0,:,:] - mean) / torch.pow(var + BN.eps,0.5) * BN.weight[0] + BN.bias[0]

print("BN_one:",BN_one)
torch.Size([8, 3, 32, 32])
tensor([[[ 1.6338e-01, -2.3648e-01, -1.2965e+00,  ...,  9.6178e-01,
          -4.1027e-01,  1.4072e+00],
         [-9.9580e-02, -8.1695e-02, -2.2693e-01,  ...,  1.1076e+00,
           2.3096e-01, -1.4278e+00],
         [ 1.2291e+00,  1.0623e+00, -5.4392e-01,  ..., -4.3424e-02,
          -2.2262e-01, -5.1729e-01],
		...,
         [-1.8724e+00,  5.1297e-01,  6.1065e-01,  ...,  1.4684e-01,
          -8.5345e-02, -1.9820e+00],
         [-1.8862e-02, -5.1397e-01, -8.9240e-01,  ...,  1.0580e+00,
           1.3407e+00,  2.6915e+00],
         [ 2.9983e-01, -8.8519e-01, -7.3509e-01,  ...,  3.3777e-01,
           4.3252e-01,  3.9348e-01]]], grad_fn=<SliceBackward0>)


torch.Size([8, 32, 32])
BN_one: tensor([[[ 1.6337e-01, -2.3647e-01, -1.2964e+00,  ...,  9.6173e-01,
          -4.1024e-01,  1.4071e+00],
         [-9.9574e-02, -8.1690e-02, -2.2692e-01,  ...,  1.1076e+00,
           2.3094e-01, -1.4277e+00],
         [ 1.2290e+00,  1.0623e+00, -5.4388e-01,  ..., -4.3422e-02,
          -2.2260e-01, -5.1726e-01],
         ...,
         [-1.8722e+00,  5.1294e-01,  6.1061e-01,  ...,  1.4683e-01,
          -8.5340e-02, -1.9818e+00],
         [-1.8860e-02, -5.1393e-01, -8.9234e-01,  ...,  1.0580e+00,
           1.3407e+00,  2.6914e+00],
         [ 2.9981e-01, -8.8513e-01, -7.3504e-01,  ...,  3.3775e-01,
           4.3249e-01,  3.9345e-01]]], grad_fn=<AddBackward0>)

2、LayerNorm

归一化维度:[C,H,W] 计算N次均值方差

import torch
from torch import nn

input = torch.randn((8,3,32,32))

LN = nn.LayerNorm((3,32,32))
output = LN(input)
print(output[0])

#test
mean = torch.mean(input[0,:,:,:])
var = torch.var(input[0,:,:,:])
print(mean)
print(var)

LN_one = ((input[0] - mean) / torch.pow(var + LN.eps,0.5)) * LN.weight[0] + LN.bias[0]
print(LN_one)

tensor([[[ 0.4859, -0.8371, -1.4560,  ...,  0.2984, -2.5241,  0.8989],
         [ 2.0582, -0.8824,  0.6683,  ...,  1.9867, -0.5992,  0.3718],
         [ 0.0882, -1.7858, -0.2839,  ...,  1.7095,  2.0336,  0.7063],
         ...,
         [-1.2601, -2.2888, -0.5641,  ...,  0.3681,  1.0542,  0.5740],
         [ 0.5747, -0.5288,  0.3245,  ...,  0.8172, -0.2308, -0.1831],
         [-1.2777, -0.1367,  0.1523,  ..., -1.9920,  1.3162, -0.3111]]],
       grad_fn=<SelectBackward0>)
tensor(0.0257)
tensor(0.9927)
tensor([[[ 0.4858, -0.8370, -1.4558,  ...,  0.2984, -2.5237,  0.8987],
         [ 2.0579, -0.8822,  0.6682,  ...,  1.9863, -0.5991,  0.3718],
         [ 0.0882, -1.7855, -0.2838,  ...,  1.7092,  2.0332,  0.7061],
         ...,
         [-1.2599, -2.2884, -0.5640,  ...,  0.3680,  1.0540,  0.5739],
         [ 0.5746, -0.5288,  0.3244,  ...,  0.8171, -0.2307, -0.1830],
         [-1.2775, -0.1367,  0.1523,  ..., -1.9917,  1.3160, -0.3110]]],
       grad_fn=<AddBackward0>)

还有一种LayerNorm,是对最后一维进行归一化

import torch
from torch import nn

#还有一种LayerNorm,是对最后一维进行归一化
input  = torch.randn((8,3,32,32))
input_ = input.permute(0,2,3,1)
# print(input_)
print(input_.shape)

LN = nn.LayerNorm(3)
output = LN(input_)
print(output[0,0,0,:])

#test
mean = torch.mean(input_[0,0,0,:])
var = torch.var(input_[0,0,0,:],unbiased=False) #注意,这里使用的是,有偏样本方差
# print(LN.weight.shape)
LN_one = ((input_[0,0,0,:] - mean) / torch.pow(var + LN.eps,0.5)) * LN.weight[0] + LN.bias[0]
print(LN_one)
torch.Size([8, 32, 32, 3])
tensor([ 1.3608, -0.3470, -1.0138], grad_fn=<SliceBackward0>)
tensor([ 1.3608, -0.3470, -1.0138], grad_fn=<AddBackward0>)

3、InstanceNorm

归一化维度:[H,W] 计算N*C次均值方差

import torch
from torch import nn

input = torch.randn((8,3,32,32))

IN = nn.InstanceNorm2d(3,affine=True) #默认没有学习参数的False
output = IN(input)

print(output[0,0,:,:])


#test
mean = torch.mean(input[0,0,:,:])
var = torch.var(input[0,0,:,:])
# print((input[0,0,:,:] - mean) / torch.pow(var + IN.eps,0.5)* IN.weight[0])
IN_one = ((input[0,0,:,:] - mean) / torch.pow(var + IN.eps,0.5)) * IN.weight[0] + IN.bias[0]
print(IN_one)
tensor([[ 0.4273,  0.9564, -1.8870,  ..., -0.6834, -2.2355,  1.1222],
        [ 0.8470,  0.0389,  1.2515,  ..., -0.7465, -0.6226, -0.4495],
        [-0.4743, -0.6062,  0.5174,  ...,  0.9884,  2.7023,  1.2147],
        ...,
        [-0.3232, -0.2927,  0.6052,  ...,  1.4219,  0.1004, -0.4636],
        [ 0.9279, -0.1547, -0.3453,  ...,  0.7218,  0.7376,  2.1859],
        [-0.8052,  1.1261, -0.3317,  ...,  0.2217, -2.0800,  1.2837]],
       grad_fn=<SliceBackward0>)
tensor([[ 0.4271,  0.9559, -1.8861,  ..., -0.6830, -2.2344,  1.1217],
        [ 0.8466,  0.0389,  1.2508,  ..., -0.7461, -0.6223, -0.4492],
        [-0.4740, -0.6059,  0.5171,  ...,  0.9879,  2.7009,  1.2141],
        ...,
        [-0.3231, -0.2926,  0.6049,  ...,  1.4212,  0.1004, -0.4634],
        [ 0.9274, -0.1546, -0.3451,  ...,  0.7214,  0.7372,  2.1849],
        [-0.8048,  1.1255, -0.3315,  ...,  0.2216, -2.0790,  1.2831]],
       grad_fn=<AddBackward0>)

4、GroupNorm

归一化维度:[N,C,H,W] -> [N,G,C//G,H,W] -> [C//G,H,W] 计算 N * G次均值方差

import torch
from torch import nn

input = torch.randn((8,6,32,32))

GN = nn.GroupNorm(3,6) #Separate 6 channels into 3 groups

output = GN(input)
print(output[0,0,:,:])

#test
input_ = input.reshape(8,3,2,32,32)
print(input_.shape)

mean = torch.mean(input_[0,0,:,:,:])
var = torch.var(input_[0,0,:,:,:])

GN_one = (input_[0,0,:,:,:] - mean) / torch.pow(var + GN.eps,0.5) * GN.weight[0] + GN.bias[0]
print(GN_one[0])
tensor([[-1.4510,  1.2567, -1.4740,  ..., -0.5039,  0.8245, -0.2772],
        [-0.0563,  0.4528,  0.7252,  ..., -1.4089, -1.6286,  0.1479],
        [-0.7159, -0.3537,  0.2684,  ..., -0.6500, -1.3845, -0.1949],
        ...,
        [-0.0898, -0.5160, -1.6077,  ..., -0.0275,  1.0162,  0.1682],
        [-1.2064,  1.1916, -0.1666,  ..., -0.0890,  2.0758, -0.5677],
        [ 2.1811,  0.0587,  0.6672,  ..., -0.0153,  0.1549, -0.2035]],
       grad_fn=<SliceBackward0>)
torch.Size([8, 3, 2, 32, 32])
tensor([[-1.4506,  1.2564, -1.4736,  ..., -0.5037,  0.8243, -0.2771],
        [-0.0563,  0.4527,  0.7250,  ..., -1.4086, -1.6282,  0.1479],
        [-0.7157, -0.3536,  0.2683,  ..., -0.6499, -1.3842, -0.1948],
        ...,
        [-0.0898, -0.5159, -1.6073,  ..., -0.0275,  1.0160,  0.1682],
        [-1.2061,  1.1913, -0.1666,  ..., -0.0889,  2.0752, -0.5676],
        [ 2.1806,  0.0587,  0.6671,  ..., -0.0153,  0.1549, -0.2035]],
       grad_fn=<SelectBackward0>)

文章出处登录后可见!

已经登录?立即刷新

共计人评分,平均

到目前为止还没有投票!成为第一位评论此文章。

(0)
上一篇 2022年5月16日 上午11:27
下一篇 2022年5月16日 上午11:31

相关推荐

本站注重文章个人版权,不会主动收集付费或者带有商业版权的文章,如果出现侵权情况只可能是作者后期更改了版权声明,如果出现这种情况请主动联系我们,我们看到会在第一时间删除!本站专注于人工智能高质量优质文章收集,方便各位学者快速找到学习资源,本站收集的文章都会附上文章出处,如果不愿意分享到本平台,我们会第一时间删除!