Pytorch Tutorial

In [1]:
%matplotlib inline 
import torch

Tensors

  • Tensorflow의 Tensor와 다르지 않다.
    • Numpy의 ndarrays를 기본적으로 활용하고 있다.
    • Numpy의 ndarrays의 대부분의 operation을 사용할 수 있도록 구성되어 있다.
  • Numpy의 operation은 CPU만을 이용해 느리지만 Tensor는 CUDA를 활용해 GPU를 이용하기 때문에 빠르게 연산을 진행할 수 있다.
In [2]:
# Construct a 5 x 3 matrix, uninitialized
x = torch.Tensor(5, 3)
print(x, '\n')

# Construct a randomly initialized matrix 
x = torch.rand(5, 3)
print(x, '\n')

# Construct a matrix with the list
x = torch.tensor([[3, 4, 5], [1, 2, 3]])
print(x, '\n')

# Get its size
print(x.size())

tensor([[ 0.0000e+00,  0.0000e+00,  1.4013e-44],
        [ 2.5223e-44,  4.2039e-45,  0.0000e+00],
        [-6.0019e-29,  3.0900e-41, -6.0019e-29],
        [ 3.0900e-41, -6.0020e-29,  3.0900e-41],
        [ 2.3822e-44,  0.0000e+00, -2.7899e-31]]) 

tensor([[0.4399, 0.6539, 0.1592],
        [0.7619, 0.6189, 0.5881],
        [0.5403, 0.4941, 0.0897],
        [0.4691, 0.6027, 0.3488],
        [0.2219, 0.4880, 0.9466]]) 

tensor([[3, 4, 5],
        [1, 2, 3]]) 

torch.Size([2, 3])

dtype and device

  • dtype - Tensor의 데이터 타입
  • device - Tensor의 작업 위치 (cpu or cuda)
In [3]:
# Construct a matrix with the list with dtype torch.float32 (FloatTensor)
x = torch.tensor([[3, 4, 5], [1, 2, 3]], dtype=torch.float32)
print(x, '\n')

# Construct a matrix with the list with default dtype (FloatTensor)
y = torch.tensor([[3., 4, 5], [1, 2, 3.]])
print(y, '\n')

# Construct a matrix with the list with default dtype (IntTensor)
z = torch.tensor([[3, 4, 5], [1, 2, 3]])
print(z, '\n')

# Success
print(x + y)

# Fail 
print(y + z)

tensor([[3., 4., 5.],
        [1., 2., 3.]]) 

tensor([[3., 4., 5.],
        [1., 2., 3.]]) 

tensor([[3, 4, 5],
        [1, 2, 3]]) 

tensor([[ 6.,  8., 10.],
        [ 2.,  4.,  6.]])
tensor([[ 6.,  8., 10.],
        [ 2.,  4.,  6.]])
In [4]:
# Construct a matrix with the list with dtype torch.float64 (DoubleTensor)
x = torch.tensor([[3, 4, 5], [1, 2, 3]], dtype=torch.float64)
print(x, '\n')

# Construct a matrix with the list with dtype torch.float32 (FloatTensor)
y = torch.tensor([[3, 4, 5], [1, 2, 3]], dtype=torch.float32)
print(y, '\n')

# Construct a matrix with the list with dtype torch.int32 (IntTensor)
z = torch.tensor([[3, 4, 5], [1, 2, 3]], dtype=torch.int32)
print(z, '\n')

# Construct a matrix with the list with dtype torch.int64 (LongTensor)
w = torch.tensor([[3, 4, 5], [1, 2, 3]], dtype=torch.int64)
print(w, '\n')

tensor([[3., 4., 5.],
        [1., 2., 3.]], dtype=torch.float64) 

tensor([[3., 4., 5.],
        [1., 2., 3.]]) 

tensor([[3, 4, 5],
        [1, 2, 3]], dtype=torch.int32) 

tensor([[3, 4, 5],
        [1, 2, 3]])
In [5]:
y = y.double() 
print(y, '\n')

print(x + y)

tensor([[3., 4., 5.],
        [1., 2., 3.]], dtype=torch.float64) 

tensor([[ 6.,  8., 10.],
        [ 2.,  4.,  6.]], dtype=torch.float64)
In [6]:
x = torch.rand(4, 5)
print(x, '\n')
print(x.device, '\n')

tensor([[0.2399, 0.9436, 0.5875, 0.1967, 0.9056],
        [0.9207, 0.5389, 0.6377, 0.5884, 0.1003],
        [0.5641, 0.1147, 0.4492, 0.5763, 0.7271],
        [0.0324, 0.7306, 0.4163, 0.7025, 0.6283]]) 

cpu
In [7]:
device = torch.device('cuda')
x = x.to(device)

print(x, '\n')
print(x.device, '\n')
print(x.double(), '\n')

tensor([[0.2399, 0.9436, 0.5875, 0.1967, 0.9056],
        [0.9207, 0.5389, 0.6377, 0.5884, 0.1003],
        [0.5641, 0.1147, 0.4492, 0.5763, 0.7271],
        [0.0324, 0.7306, 0.4163, 0.7025, 0.6283]], device='cuda:0') 

cuda:0 

tensor([[0.2399, 0.9436, 0.5875, 0.1967, 0.9056],
        [0.9207, 0.5389, 0.6377, 0.5884, 0.1003],
        [0.5641, 0.1147, 0.4492, 0.5763, 0.7271],
        [0.0324, 0.7306, 0.4163, 0.7025, 0.6283]], device='cuda:0',
       dtype=torch.float64)
In [8]:
device = torch.device('cuda')

x = torch.randn(4, 3, dtype=torch.float64)
y = torch.randn(4, 3, dtype=torch.float32)
z = torch.randint(0, 10, (4, 3), dtype=torch.int32)

z = z.to(device)

print('Before "to" method\n')

print(x.dtype, x.device)
print(y.dtype, y.device)
print(z.dtype, z.device, '\n')

print('After "to" method\n')

# to method with specific dtype and device 
x = x.to(dtype=torch.int32, device=device)

# to method with some tensor 
y = y.to(z) # z랑계산하게 z랑 똑같이 맞춰줘!

print(x.dtype, x.device)
print(y.dtype, y.device)
print(z.dtype, z.device, '\n')

Before "to" method

torch.float64 cpu
torch.float32 cpu
torch.int32 cuda:0 

After "to" method

torch.int32 cuda:0
torch.int32 cuda:0
torch.int32 cuda:0
In [9]:
# Use specific GPU 
device1 = torch.device('cuda') 
device2 = torch.device('cuda:1')

x = torch.randn(4, 3) 
y = torch.randn(4, 3) 

x = x.to(device1)
y = y.to(device2)

print(x.device, y.device)
print(x + y)
#같은 쿠다 안에있어야지만 연산가능

cuda:0 cuda:1

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-9-f739c620b345> in <module>
     10 
     11 print(x.device, y.device)
---> 12 print(x + y)
     13 #같은 쿠다 안에있어야지만 연산가능

RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cuda:1!
In [ ]:
# NOT RECOMMENDED  # .to(device) 를 쓰도록!
x = torch.randn(4, 3)
print(x, x.device, '\n')

x = x.cuda()
print(x, x.device, '\n')

x = x.cpu()
print(x, x.device, '\n')

Construct a matrix with specific value

In [ ]:
# Construct a 3 x 5 matrix, uninitialized 
x = torch.empty(3, 5)
print(x, '\n')

# Construct a 3 x 5 matrix with zeros 
x = torch.zeros(3, 5)
print(x, '\n')

# Construct a 3 x 5 matrix with ones
x = torch.ones(3, 5)
print(x, '\n')

# Construct a 3 x 5 matrix with given value 
x = torch.full((3,5), 3.1415)
print(x, '\n')
In [ ]:
# Construct a vector with uniformly spaced values in a given range 
x = torch.arange(2, 10, 2) 
print(x, '\n')

y = torch.linspace(0, 5, 3) #0부터 5사이에 9개 등차수열로 
print(y, '\n')

# Construct a vector with logarithmically spaced values in a given range
z = torch.logspace(-10, 10, 5) # 10^-10부터 10^10 까지 리니어하게 등비수열 5칸
print(z, '\n')

# Construct a identity matrix 
z = torch.eye(5)
print(z, '\n')

Construct a matrix with random value

In [ ]:
# Construct a 3 x 5 matrix with random value from uniform distribution, i.e. Uniform[0, 1)
x = torch.rand(3, 5)
print(x, '\n')

# Construct a 3 x 5 matrix with random value from normal distribution, i.e. Normal(0, 1)
x = torch.randn(3, 5)
print(x, '\n')

# Construct a 3 x 5 matrix filled with random integers, i.e. Uniform[low, high)
x = torch.randint(3, 10, (3, 5))
print(x, '\n')

# Construct a vector with random permutation 
x = torch.randperm(9)
print(x, '\n')

*_like function and new_* function

  • *_like: Tensor를 input으로 받아, Tensor 모양의 matrix를 return.
  • new_*: Shape를 input으로 받아, Tensor와 같은 type과 device를 가지는 matrix를 return
In [ ]:
x = torch.randint(3, (2,3)).cuda()
print(x, '\n')

y=  torch.zeros_like(x)
print(y, '\n')
In [10]:
x = torch.rand(2, 3)
print(x, '\n')

y = torch.zeros_like(x)
print(y, '\n')

tensor([[0.0174, 0.1490, 0.0478],
        [0.7532, 0.5263, 0.5427]]) 

tensor([[0., 0., 0.],
        [0., 0., 0.]])
In [11]:
x = torch.rand(2,3).int()
print(x, '\n')
y = torch.randint_like(x, 2,5)
print(y, '\n')

tensor([[0, 0, 0],
        [0, 0, 0]], dtype=torch.int32) 

tensor([[4, 4, 2],
        [3, 3, 3]], dtype=torch.int32)
In [12]:
device = torch.device('cuda') 
x = torch.rand(2, 3)
x = x.to(device) 

y = torch.zeros_like(x)
print(x, '\n')
print(y, '\n')
#~_like 는 다따라가고
#뉴제로스는 shape맘대로

tensor([[0.8075, 0.2560, 0.0266],
        [0.2092, 0.7819, 0.7184]], device='cuda:0') 

tensor([[0., 0., 0.],
        [0., 0., 0.]], device='cuda:0')
In [13]:
device = torch.device('cuda')

x = torch.randint(0, 10, (3, 4), dtype=torch.int32)
y = torch.randint(0, 10, (3, 4), dtype=torch.float32, device=device)

# Make zero matrix with attribute of x
z = x.new_zeros(2, 3)
print(z, '\n')
print(z.dtype, z.device,'\n')

# Make zero matrix with attribute of y 
z = y.new_zeros(2, 3)
print(z, '\n')
print(z.dtype, z.device, '\n')

tensor([[0, 0, 0],
        [0, 0, 0]], dtype=torch.int32) 

torch.int32 cpu 

tensor([[0., 0., 0.],
        [0., 0., 0.]], device='cuda:0') 

torch.float32 cuda:0

Numpy Bridge

In [14]:
import numpy as np

a = np.ones(5)
b = torch.from_numpy(a)
print(a)
print(b)
print(" ")
np.add(a, 1, out=a)
print(a)
print(b)

torch.add(b, 1, out=b)
print(a)
print(b)

[1. 1. 1. 1. 1.]
tensor([1., 1., 1., 1., 1.], dtype=torch.float64)
 
[2. 2. 2. 2. 2.]
tensor([2., 2., 2., 2., 2.], dtype=torch.float64)
[3. 3. 3. 3. 3.]
tensor([3., 3., 3., 3., 3.], dtype=torch.float64)
In [15]:
a = torch.ones(5) 
b = a.numpy()
print(b) 

[1. 1. 1. 1. 1.]
In [16]:
# Caution: the variable on GPU cannot convert to NumPy Array. 
a = torch.ones(5).to(torch.device('cuda')) 
b = a.numpy() # error 

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-16-f29df78e70d0> in <module>
      1 # Caution: the variable on GPU cannot convert to NumPy Array.
      2 a = torch.ones(5).to(torch.device('cuda'))
----> 3 b = a.numpy() # error

TypeError: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.

Operations

In [17]:
# Addition: syntax 1
x = torch.rand(5, 3) 
y = torch.rand(5, 3)
print(x + y, '\n')

# Addition: syntax 2
print(torch.add(x, y), '\n')

# Addition: giving an output tensor
result = torch.Tensor(5, 3)
torch.add(x, y, out=result)
print(result, '\n')

# Addition: in-place
y.add_(x) # (y = y + x)
print(y, '\n')

tensor([[1.1580, 0.6293, 0.4965],
        [1.2938, 1.0122, 0.9994],
        [1.4383, 0.4632, 1.2258],
        [1.1884, 1.4669, 1.2661],
        [0.5021, 0.3039, 0.8037]]) 

tensor([[1.1580, 0.6293, 0.4965],
        [1.2938, 1.0122, 0.9994],
        [1.4383, 0.4632, 1.2258],
        [1.1884, 1.4669, 1.2661],
        [0.5021, 0.3039, 0.8037]]) 

tensor([[1.1580, 0.6293, 0.4965],
        [1.2938, 1.0122, 0.9994],
        [1.4383, 0.4632, 1.2258],
        [1.1884, 1.4669, 1.2661],
        [0.5021, 0.3039, 0.8037]]) 

tensor([[1.1580, 0.6293, 0.4965],
        [1.2938, 1.0122, 0.9994],
        [1.4383, 0.4632, 1.2258],
        [1.1884, 1.4669, 1.2661],
        [0.5021, 0.3039, 0.8037]])
In [18]:
# You can use standard numpy-like indexing with all bells and whistles
print(x)
print(x[:, 1], '\n')
print(x[x > 0.5])

tensor([[0.9431, 0.2065, 0.4283],
        [0.6476, 0.6180, 0.8714],
        [0.4833, 0.3555, 0.2892],
        [0.7041, 0.4933, 0.5038],
        [0.4716, 0.2753, 0.3028]])
tensor([0.2065, 0.6180, 0.3555, 0.4933, 0.2753]) 

tensor([0.9431, 0.6476, 0.6180, 0.8714, 0.7041, 0.5038])
In [19]:
x = torch.randn(2, 3) 
y = torch.randn(2, 3)
z = torch.randn(3, 2)

print('Pre-define\n')
print(x, '\n')
print(y, '\n')
print(z, '\n')

Pre-define

tensor([[-0.4528,  1.6292,  0.6487],
        [ 1.2521, -0.0269,  0.1381]]) 

tensor([[ 1.0928,  0.0441,  0.3369],
        [-1.5654,  1.5711,  2.7788]]) 

tensor([[-0.5939, -1.1290],
        [ 0.2327, -1.6666],
        [ 2.5601, -0.1138]])
In [20]:
print('Operation\n')
# Add 
print(x + y, '\n')

# Sub 
print(x - y, '\n')

# Element-wise Mul
print(x * y, '\n')

# Element-wise Div
print(x / y, '\n')

# Matrix Mul.
print(x @ z, '\n')
print(torch.matmul(x, z))

Operation

tensor([[ 0.6399,  1.6734,  0.9856],
        [-0.3133,  1.5442,  2.9168]]) 

tensor([[-1.5456,  1.5851,  0.3118],
        [ 2.8176, -1.5980, -2.6407]]) 

tensor([[-0.4949,  0.0719,  0.2186],
        [-1.9601, -0.0422,  0.3836]]) 

tensor([[-4.1441e-01,  3.6908e+01,  1.9255e+00],
        [-7.9988e-01, -1.7114e-02,  4.9680e-02]]) 

tensor([[ 2.3089, -2.2779],
        [-0.3965, -1.3846]]) 

tensor([[ 2.3089, -2.2779],
        [-0.3965, -1.3846]])
In [21]:
# Change the shape of tensor 
x = torch.arange(0, 10)
print(x, '\n')

y = x.view(2, 5)
print(y, '\n')

x = torch.arange(0, 30).view(5, 6)
print(x, '\n')
print(x.size(), '\n')

y = x.view(-1, 2, 5) # of element : 30  # -1 와일드카드 그냥 계산 딱맞을때만 
print(y, '\n')       # ? *2 * 5 
print(y.size(), '\n')

tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) 

tensor([[0, 1, 2, 3, 4],
        [5, 6, 7, 8, 9]]) 

tensor([[ 0,  1,  2,  3,  4,  5],
        [ 6,  7,  8,  9, 10, 11],
        [12, 13, 14, 15, 16, 17],
        [18, 19, 20, 21, 22, 23],
        [24, 25, 26, 27, 28, 29]]) 

torch.Size([5, 6]) 

tensor([[[ 0,  1,  2,  3,  4],
         [ 5,  6,  7,  8,  9]],

        [[10, 11, 12, 13, 14],
         [15, 16, 17, 18, 19]],

        [[20, 21, 22, 23, 24],
         [25, 26, 27, 28, 29]]]) 

torch.Size([3, 2, 5])
In [22]:
# Change the dimension of tensor
x = torch.arange(0, 10).view(2, 5)
print(x, '\n')

y = x.permute(1, 0) #transpose
print(y, '\n')

y = x.t()
print(y, '\n') # 2차원까지만가능

tensor([[0, 1, 2, 3, 4],
        [5, 6, 7, 8, 9]]) 

tensor([[0, 5],
        [1, 6],
        [2, 7],
        [3, 8],
        [4, 9]]) 

tensor([[0, 5],
        [1, 6],
        [2, 7],
        [3, 8],
        [4, 9]])
In [24]:
# Add the dimension of tensor 
print(x)
z = x.unsqueeze(0)   (2, 5) -> (1,2,5)
print(z)
print(z.size(), '\n')

tensor([[0, 1, 2, 3, 4],
        [5, 6, 7, 8, 9]])
tensor([[[0, 1, 2, 3, 4],
         [5, 6, 7, 8, 9]]])
torch.Size([1, 2, 5])
In [29]:
z = x.unsqueeze(1)
print(z)
print(z.size(), '\n')

tensor([[[0, 1, 2, 3, 4]],

        [[5, 6, 7, 8, 9]]])
torch.Size([2, 1, 5])
In [30]:
# Remove the unnecessary dimension of tensor (size=1)
z = z.squeeze()
print(z.size(), '\n')

torch.Size([2, 5])
In [37]:
a = torch.arange(9).view(3,3)
b = torch.arange(3).view(3,1)
print(a)
print(b)

tensor([[0, 1, 2],
        [3, 4, 5],
        [6, 7, 8]])
tensor([[0],
        [1],
        [2]])
In [38]:
print(a+b)

tensor([[ 0,  1,  2],
        [ 4,  5,  6],
        [ 8,  9, 10]])
In [60]:
import time

a= torch.arange(10000).view(100, 100)
b = torch.arange(100)
start = time.time()
c = a+b.expand(100,100)
end = time.time()
print(end - start)

start = time.time()
c=a+b
end = time.time()
print(end - start)

0.0003573894500732422
0.00026226043701171875
In [ ]: