diff --git "a/Lesson1-\345\210\235\350\247\201PyTorch/autograd_demo.py" "b/Lesson1-\345\210\235\350\247\201PyTorch/autograd_demo.py" new file mode 100644 index 0000000..3e5958d --- /dev/null +++ "b/Lesson1-\345\210\235\350\247\201PyTorch/autograd_demo.py" @@ -0,0 +1,14 @@ +import torch +from torch import autograd + + +x = torch.tensor(1.) +a = torch.tensor(1., requires_grad=True) +b = torch.tensor(2., requires_grad=True) +c = torch.tensor(3., requires_grad=True) + +y = a**2 * x + b * x + c + +print('before:', a.grad, b.grad, c.grad) +grads = autograd.grad(y, [a, b, c]) +print('after :', grads[0], grads[1], grads[2]) \ No newline at end of file diff --git "a/Lesson1-\345\210\235\350\247\201PyTorch/demo.lua" "b/Lesson1-\345\210\235\350\247\201PyTorch/demo.lua" new file mode 100644 index 0000000..0cf09a7 --- /dev/null +++ "b/Lesson1-\345\210\235\350\247\201PyTorch/demo.lua" @@ -0,0 +1,11 @@ +torch.Tensor{1,2,3} + +function gradUpdate(mlp,x,y,learningRate) + local criterion = nn.ClassNLLCriterion() + pred = mlp:forward(x) + local err = criterion:forward(pred, y); + mlp:zeroGradParameters(); + local t = criterion:backward(pred, y); + mlp:backward(x, t); + mlp:updateParameters(learningRate); +end diff --git "a/Lesson1-\345\210\235\350\247\201PyTorch/lesson1.pdf" "b/Lesson1-\345\210\235\350\247\201PyTorch/lesson1.pdf" new file mode 100644 index 0000000..7d29dbc Binary files /dev/null and "b/Lesson1-\345\210\235\350\247\201PyTorch/lesson1.pdf" differ diff --git "a/Lesson1-\345\210\235\350\247\201PyTorch/lesson1.py" "b/Lesson1-\345\210\235\350\247\201PyTorch/lesson1.py" new file mode 100644 index 0000000..6cd8c01 --- /dev/null +++ "b/Lesson1-\345\210\235\350\247\201PyTorch/lesson1.py" @@ -0,0 +1,29 @@ +import torch +import time +print(torch.__version__) +print(torch.cuda.is_available()) +# print('hello, world.') + + +a = torch.randn(10000, 1000) +b = torch.randn(1000, 2000) + +t0 = time.time() +c = torch.matmul(a, b) +t1 = time.time() +print(a.device, t1 - t0, c.norm(2)) + +device = torch.device('cuda') +a = a.to(device) +b = b.to(device) + +t0 = time.time() +c = torch.matmul(a, b) +t2 = time.time() +print(a.device, t2 - t0, c.norm(2)) + +t0 = time.time() +c = torch.matmul(a, b) +t2 = time.time() +print(a.device, t2 - t0, c.norm(2)) + diff --git "a/Lesson2-\345\274\200\345\217\221\347\216\257\345\242\203\345\207\206\345\244\207/lesson2.pdf" "b/Lesson2-\345\274\200\345\217\221\347\216\257\345\242\203\345\207\206\345\244\207/lesson2.pdf" new file mode 100644 index 0000000..94698df Binary files /dev/null and "b/Lesson2-\345\274\200\345\217\221\347\216\257\345\242\203\345\207\206\345\244\207/lesson2.pdf" differ diff --git "a/Lesson2-\345\274\200\345\217\221\347\216\257\345\242\203\345\207\206\345\244\207/main.py" "b/Lesson2-\345\274\200\345\217\221\347\216\257\345\242\203\345\207\206\345\244\207/main.py" new file mode 100644 index 0000000..4ea5141 --- /dev/null +++ "b/Lesson2-\345\274\200\345\217\221\347\216\257\345\242\203\345\207\206\345\244\207/main.py" @@ -0,0 +1,4 @@ +import torch + +print(torch.__version__) +print('gpu:', torch.cuda.is_available()) \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..afe0948 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# DeepLearningTutorials diff --git a/lesson10-Broadcasting/Broadcasting.pdf b/lesson10-Broadcasting/Broadcasting.pdf new file mode 100644 index 0000000..0a1368f Binary files /dev/null and b/lesson10-Broadcasting/Broadcasting.pdf differ diff --git "a/lesson10-Broadcasting/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" "b/lesson10-Broadcasting/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" new file mode 100644 index 0000000..e69de29 diff --git "a/lesson11-Tensor\345\220\210\345\271\266\344\270\216\345\210\206\345\211\262/Concat.pdf" "b/lesson11-Tensor\345\220\210\345\271\266\344\270\216\345\210\206\345\211\262/Concat.pdf" new file mode 100644 index 0000000..2445351 Binary files /dev/null and "b/lesson11-Tensor\345\220\210\345\271\266\344\270\216\345\210\206\345\211\262/Concat.pdf" differ diff --git "a/lesson11-Tensor\345\220\210\345\271\266\344\270\216\345\210\206\345\211\262/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" "b/lesson11-Tensor\345\220\210\345\271\266\344\270\216\345\210\206\345\211\262/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" new file mode 100644 index 0000000..e69de29 diff --git "a/lesson12-Tensor\350\277\220\347\256\227/math operation.pdf" "b/lesson12-Tensor\350\277\220\347\256\227/math operation.pdf" new file mode 100644 index 0000000..6301eb8 Binary files /dev/null and "b/lesson12-Tensor\350\277\220\347\256\227/math operation.pdf" differ diff --git "a/lesson12-Tensor\350\277\220\347\256\227/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" "b/lesson12-Tensor\350\277\220\347\256\227/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" new file mode 100644 index 0000000..e69de29 diff --git "a/lesson13-Tensor\347\273\237\350\256\241/statisics.pdf" "b/lesson13-Tensor\347\273\237\350\256\241/statisics.pdf" new file mode 100644 index 0000000..a0e4051 Binary files /dev/null and "b/lesson13-Tensor\347\273\237\350\256\241/statisics.pdf" differ diff --git "a/lesson13-Tensor\347\273\237\350\256\241/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" "b/lesson13-Tensor\347\273\237\350\256\241/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" new file mode 100644 index 0000000..e69de29 diff --git "a/lesson14-Tensor\351\253\230\351\230\266/tensor advanced.pdf" "b/lesson14-Tensor\351\253\230\351\230\266/tensor advanced.pdf" new file mode 100644 index 0000000..3e65d3f Binary files /dev/null and "b/lesson14-Tensor\351\253\230\351\230\266/tensor advanced.pdf" differ diff --git "a/lesson14-Tensor\351\253\230\351\230\266/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" "b/lesson14-Tensor\351\253\230\351\230\266/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" new file mode 100644 index 0000000..e69de29 diff --git "a/lesson16-\344\273\200\344\271\210\346\230\257\346\242\257\345\272\246/16 \346\242\257\345\272\246.pdf" "b/lesson16-\344\273\200\344\271\210\346\230\257\346\242\257\345\272\246/16 \346\242\257\345\272\246.pdf" new file mode 100644 index 0000000..1b36255 Binary files /dev/null and "b/lesson16-\344\273\200\344\271\210\346\230\257\346\242\257\345\272\246/16 \346\242\257\345\272\246.pdf" differ diff --git "a/lesson17-\345\270\270\350\247\201\346\242\257\345\272\246/17 \345\270\270\350\247\201\345\207\275\346\225\260\346\242\257\345\272\246.pdf" "b/lesson17-\345\270\270\350\247\201\346\242\257\345\272\246/17 \345\270\270\350\247\201\345\207\275\346\225\260\346\242\257\345\272\246.pdf" new file mode 100644 index 0000000..768092d Binary files /dev/null and "b/lesson17-\345\270\270\350\247\201\346\242\257\345\272\246/17 \345\270\270\350\247\201\345\207\275\346\225\260\346\242\257\345\272\246.pdf" differ diff --git "a/lesson18-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216Loss\347\232\204\346\242\257\345\272\246/18.1 \346\277\200\346\264\273\345\207\275\346\225\260\346\242\257\345\272\246.pdf" "b/lesson18-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216Loss\347\232\204\346\242\257\345\272\246/18.1 \346\277\200\346\264\273\345\207\275\346\225\260\346\242\257\345\272\246.pdf" new file mode 100644 index 0000000..86d0ef9 Binary files /dev/null and "b/lesson18-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216Loss\347\232\204\346\242\257\345\272\246/18.1 \346\277\200\346\264\273\345\207\275\346\225\260\346\242\257\345\272\246.pdf" differ diff --git "a/lesson18-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216Loss\347\232\204\346\242\257\345\272\246/18.2 LOSS\345\217\212\345\205\266\346\242\257\345\272\246.pdf" "b/lesson18-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216Loss\347\232\204\346\242\257\345\272\246/18.2 LOSS\345\217\212\345\205\266\346\242\257\345\272\246.pdf" new file mode 100644 index 0000000..a095827 Binary files /dev/null and "b/lesson18-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216Loss\347\232\204\346\242\257\345\272\246/18.2 LOSS\345\217\212\345\205\266\346\242\257\345\272\246.pdf" differ diff --git "a/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/19.1 \345\215\225\344\270\200\350\276\223\345\207\272\346\204\237\347\237\245\346\234\272.pdf" "b/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/19.1 \345\215\225\344\270\200\350\276\223\345\207\272\346\204\237\347\237\245\346\234\272.pdf" new file mode 100644 index 0000000..f634b05 Binary files /dev/null and "b/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/19.1 \345\215\225\344\270\200\350\276\223\345\207\272\346\204\237\347\237\245\346\234\272.pdf" differ diff --git "a/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/19.2 \345\244\232\350\276\223\345\207\272\346\204\237\347\237\245\346\234\272.pdf" "b/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/19.2 \345\244\232\350\276\223\345\207\272\346\204\237\347\237\245\346\234\272.pdf" new file mode 100644 index 0000000..480a89d Binary files /dev/null and "b/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/19.2 \345\244\232\350\276\223\345\207\272\346\204\237\347\237\245\346\234\272.pdf" differ diff --git "a/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/lesson19.1.mp4.baiduyun.uploading.cfg" "b/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/lesson19.1.mp4.baiduyun.uploading.cfg" new file mode 100644 index 0000000..d6eef82 Binary files /dev/null and "b/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/lesson19.1.mp4.baiduyun.uploading.cfg" differ diff --git "a/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/lesson19.2.mp4.baiduyun.uploading.cfg" "b/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/lesson19.2.mp4.baiduyun.uploading.cfg" new file mode 100644 index 0000000..c7da486 Binary files /dev/null and "b/lesson19-\346\204\237\347\237\245\346\234\272\347\232\204\346\242\257\345\272\246\346\216\250\345\257\274/lesson19.2.mp4.baiduyun.uploading.cfg" differ diff --git "a/lesson20-\351\223\276\345\274\217\346\263\225\345\210\231/20.pdf" "b/lesson20-\351\223\276\345\274\217\346\263\225\345\210\231/20.pdf" new file mode 100644 index 0000000..4b13dcc Binary files /dev/null and "b/lesson20-\351\223\276\345\274\217\346\263\225\345\210\231/20.pdf" differ diff --git "a/lesson21-MLP\345\217\215\345\220\221\344\274\240\346\222\255\346\216\250\345\257\274/21.pdf" "b/lesson21-MLP\345\217\215\345\220\221\344\274\240\346\222\255\346\216\250\345\257\274/21.pdf" new file mode 100644 index 0000000..dcf253c Binary files /dev/null and "b/lesson21-MLP\345\217\215\345\220\221\344\274\240\346\222\255\346\216\250\345\257\274/21.pdf" differ diff --git "a/lesson22-\344\274\230\345\214\226\345\260\217\345\256\236\344\276\213/22 Himmelblau.pdf" "b/lesson22-\344\274\230\345\214\226\345\260\217\345\256\236\344\276\213/22 Himmelblau.pdf" new file mode 100644 index 0000000..5a71c49 Binary files /dev/null and "b/lesson22-\344\274\230\345\214\226\345\260\217\345\256\236\344\276\213/22 Himmelblau.pdf" differ diff --git "a/lesson22-\344\274\230\345\214\226\345\260\217\345\256\236\344\276\213/main.py" "b/lesson22-\344\274\230\345\214\226\345\260\217\345\256\236\344\276\213/main.py" new file mode 100644 index 0000000..b9364a0 --- /dev/null +++ "b/lesson22-\344\274\230\345\214\226\345\260\217\345\256\236\344\276\213/main.py" @@ -0,0 +1,41 @@ +import numpy as np +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import pyplot as plt +import torch + + + +def himmelblau(x): + return (x[0] ** 2 + x[1] - 11) ** 2 + (x[0] + x[1] ** 2 - 7) ** 2 + + +x = np.arange(-6, 6, 0.1) +y = np.arange(-6, 6, 0.1) +print('x,y range:', x.shape, y.shape) +X, Y = np.meshgrid(x, y) +print('X,Y maps:', X.shape, Y.shape) +Z = himmelblau([X, Y]) + +fig = plt.figure('himmelblau') +ax = fig.gca(projection='3d') +ax.plot_surface(X, Y, Z) +ax.view_init(60, -30) +ax.set_xlabel('x') +ax.set_ylabel('y') +plt.show() + + +# [1., 0.], [-4, 0.], [4, 0.] +x = torch.tensor([-4., 0.], requires_grad=True) +optimizer = torch.optim.Adam([x], lr=1e-3) +for step in range(20000): + + pred = himmelblau(x) + + optimizer.zero_grad() + pred.backward() + optimizer.step() + + if step % 2000 == 0: + print ('step {}: x = {}, f(x) = {}' + .format(step, x.tolist(), pred.item())) diff --git a/lesson24-Logistic Regression/24 LR.pdf b/lesson24-Logistic Regression/24 LR.pdf new file mode 100644 index 0000000..af6faa4 Binary files /dev/null and b/lesson24-Logistic Regression/24 LR.pdf differ diff --git "a/lesson25-\344\272\244\345\217\211\347\206\265/25 \344\272\244\345\217\211\347\206\265.pdf" "b/lesson25-\344\272\244\345\217\211\347\206\265/25 \344\272\244\345\217\211\347\206\265.pdf" new file mode 100644 index 0000000..96cea4e Binary files /dev/null and "b/lesson25-\344\272\244\345\217\211\347\206\265/25 \344\272\244\345\217\211\347\206\265.pdf" differ diff --git "a/lesson26-LR\345\244\232\345\210\206\347\261\273\345\256\236\346\210\230/26.pdf" "b/lesson26-LR\345\244\232\345\210\206\347\261\273\345\256\236\346\210\230/26.pdf" new file mode 100644 index 0000000..7791592 Binary files /dev/null and "b/lesson26-LR\345\244\232\345\210\206\347\261\273\345\256\236\346\210\230/26.pdf" differ diff --git "a/lesson26-LR\345\244\232\345\210\206\347\261\273\345\256\236\346\210\230/main.py" "b/lesson26-LR\345\244\232\345\210\206\347\261\273\345\256\236\346\210\230/main.py" new file mode 100644 index 0000000..0463445 --- /dev/null +++ "b/lesson26-LR\345\244\232\345\210\206\347\261\273\345\256\236\346\210\230/main.py" @@ -0,0 +1,86 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms + + +batch_size=200 +learning_rate=0.01 +epochs=10 + +train_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) +test_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) + + + +w1, b1 = torch.randn(200, 784, requires_grad=True),\ + torch.zeros(200, requires_grad=True) +w2, b2 = torch.randn(200, 200, requires_grad=True),\ + torch.zeros(200, requires_grad=True) +w3, b3 = torch.randn(10, 200, requires_grad=True),\ + torch.zeros(10, requires_grad=True) + +torch.nn.init.kaiming_normal_(w1) +torch.nn.init.kaiming_normal_(w2) +torch.nn.init.kaiming_normal_(w3) + + +def forward(x): + x = x@w1.t() + b1 + x = F.relu(x) + x = x@w2.t() + b2 + x = F.relu(x) + x = x@w3.t() + b3 + x = F.relu(x) + return x + + + +optimizer = optim.SGD([w1, b1, w2, b2, w3, b3], lr=learning_rate) +criteon = nn.CrossEntropyLoss() + +for epoch in range(epochs): + + for batch_idx, (data, target) in enumerate(train_loader): + data = data.view(-1, 28*28) + + logits = forward(data) + loss = criteon(logits, target) + + optimizer.zero_grad() + loss.backward() + # print(w1.grad.norm(), w2.grad.norm()) + optimizer.step() + + if batch_idx % 100 == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + + test_loss = 0 + correct = 0 + for data, target in test_loader: + data = data.view(-1, 28 * 28) + logits = forward(data) + test_loss += criteon(logits, target).item() + + pred = logits.data.max(1)[1] + correct += pred.eq(target.data).sum() + + test_loss /= len(test_loader.dataset) + print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) diff --git "a/lesson27-MLP\347\275\221\347\273\234\345\261\202/27 \345\205\250\350\277\236\346\216\245\345\261\202.pdf" "b/lesson27-MLP\347\275\221\347\273\234\345\261\202/27 \345\205\250\350\277\236\346\216\245\345\261\202.pdf" new file mode 100644 index 0000000..e7b8fe8 Binary files /dev/null and "b/lesson27-MLP\347\275\221\347\273\234\345\261\202/27 \345\205\250\350\277\236\346\216\245\345\261\202.pdf" differ diff --git "a/lesson27-MLP\347\275\221\347\273\234\345\261\202/main.py" "b/lesson27-MLP\347\275\221\347\273\234\345\261\202/main.py" new file mode 100644 index 0000000..95d430e --- /dev/null +++ "b/lesson27-MLP\347\275\221\347\273\234\345\261\202/main.py" @@ -0,0 +1,83 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms + + +batch_size=200 +learning_rate=0.01 +epochs=10 + +train_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) +test_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) + + + +class MLP(nn.Module): + + def __init__(self): + super(MLP, self).__init__() + + self.model = nn.Sequential( + nn.Linear(784, 200), + nn.ReLU(inplace=True), + nn.Linear(200, 200), + nn.ReLU(inplace=True), + nn.Linear(200, 10), + nn.ReLU(inplace=True), + ) + + def forward(self, x): + x = self.model(x) + + return x + +net = MLP() +optimizer = optim.SGD(net.parameters(), lr=learning_rate) +criteon = nn.CrossEntropyLoss() + +for epoch in range(epochs): + + for batch_idx, (data, target) in enumerate(train_loader): + data = data.view(-1, 28*28) + + logits = net(data) + loss = criteon(logits, target) + + optimizer.zero_grad() + loss.backward() + # print(w1.grad.norm(), w2.grad.norm()) + optimizer.step() + + if batch_idx % 100 == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + + test_loss = 0 + correct = 0 + for data, target in test_loader: + data = data.view(-1, 28 * 28) + logits = net(data) + test_loss += criteon(logits, target).item() + + pred = logits.data.max(1)[1] + correct += pred.eq(target.data).sum() + + test_loss /= len(test_loader.dataset) + print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) diff --git "a/lesson28-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216GPU\345\212\240\351\200\237/28.pdf" "b/lesson28-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216GPU\345\212\240\351\200\237/28.pdf" new file mode 100644 index 0000000..22b91a6 Binary files /dev/null and "b/lesson28-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216GPU\345\212\240\351\200\237/28.pdf" differ diff --git "a/lesson28-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216GPU\345\212\240\351\200\237/main.py" "b/lesson28-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216GPU\345\212\240\351\200\237/main.py" new file mode 100644 index 0000000..928f815 --- /dev/null +++ "b/lesson28-\346\277\200\346\264\273\345\207\275\346\225\260\344\270\216GPU\345\212\240\351\200\237/main.py" @@ -0,0 +1,89 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms + + +batch_size=200 +learning_rate=0.01 +epochs=10 + +train_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) +test_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) + + + + + + +class MLP(nn.Module): + + def __init__(self): + super(MLP, self).__init__() + + self.model = nn.Sequential( + nn.Linear(784, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 10), + nn.LeakyReLU(inplace=True), + ) + + def forward(self, x): + x = self.model(x) + + return x + +device = torch.device('cuda:0') +net = MLP().to(device) +optimizer = optim.SGD(net.parameters(), lr=learning_rate) +criteon = nn.CrossEntropyLoss().to(device) + +for epoch in range(epochs): + + for batch_idx, (data, target) in enumerate(train_loader): + data = data.view(-1, 28*28) + data, target = data.to(device), target.cuda() + + logits = net(data) + loss = criteon(logits, target) + + optimizer.zero_grad() + loss.backward() + # print(w1.grad.norm(), w2.grad.norm()) + optimizer.step() + + if batch_idx % 100 == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + + test_loss = 0 + correct = 0 + for data, target in test_loader: + data = data.view(-1, 28 * 28) + data, target = data.to(device), target.cuda() + logits = net(data) + test_loss += criteon(logits, target).item() + + pred = logits.data.max(1)[1] + correct += pred.eq(target.data).sum() + + test_loss /= len(test_loader.dataset) + print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) diff --git "a/lesson29-MNIST\346\265\213\350\257\225/29.pdf" "b/lesson29-MNIST\346\265\213\350\257\225/29.pdf" new file mode 100644 index 0000000..6357b40 Binary files /dev/null and "b/lesson29-MNIST\346\265\213\350\257\225/29.pdf" differ diff --git "a/lesson29-MNIST\346\265\213\350\257\225/main.py" "b/lesson29-MNIST\346\265\213\350\257\225/main.py" new file mode 100644 index 0000000..8043753 --- /dev/null +++ "b/lesson29-MNIST\346\265\213\350\257\225/main.py" @@ -0,0 +1,103 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms + +from visdom import Visdom + +batch_size=200 +learning_rate=0.01 +epochs=10 + +train_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + # transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) +test_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + # transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) + + + +class MLP(nn.Module): + + def __init__(self): + super(MLP, self).__init__() + + self.model = nn.Sequential( + nn.Linear(784, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 10), + nn.LeakyReLU(inplace=True), + ) + + def forward(self, x): + x = self.model(x) + + return x + +device = torch.device('cuda:0') +net = MLP().to(device) +optimizer = optim.SGD(net.parameters(), lr=learning_rate) +criteon = nn.CrossEntropyLoss().to(device) + +viz = Visdom() + +viz.line([0.], [0.], win='train_loss', opts=dict(title='train loss')) +viz.line([[0.0, 0.0]], [0.], win='test', opts=dict(title='test loss&acc.', + legend=['loss', 'acc.'])) +global_step = 0 + +for epoch in range(epochs): + + for batch_idx, (data, target) in enumerate(train_loader): + data = data.view(-1, 28*28) + data, target = data.to(device), target.cuda() + + logits = net(data) + loss = criteon(logits, target) + + optimizer.zero_grad() + loss.backward() + # print(w1.grad.norm(), w2.grad.norm()) + optimizer.step() + + global_step += 1 + viz.line([loss.item()], [global_step], win='train_loss', update='append') + + if batch_idx % 100 == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + + test_loss = 0 + correct = 0 + for data, target in test_loader: + data = data.view(-1, 28 * 28) + data, target = data.to(device), target.cuda() + logits = net(data) + test_loss += criteon(logits, target).item() + + pred = logits.argmax(dim=1) + correct += pred.eq(target).float().sum().item() + + viz.line([[test_loss, correct / len(test_loader.dataset)]], + [global_step], win='test', update='append') + viz.images(data.view(-1, 1, 28, 28), win='x') + viz.text(str(pred.detach().cpu().numpy()), win='pred', + opts=dict(title='pred')) + + test_loss /= len(test_loader.dataset) + print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) diff --git "a/lesson3-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213/lesson3.pdf" "b/lesson3-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213/lesson3.pdf" new file mode 100644 index 0000000..e100e46 Binary files /dev/null and "b/lesson3-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213/lesson3.pdf" differ diff --git "a/lesson30-Visdom\345\217\257\350\247\206\345\214\226/30 \345\217\257\350\247\206\345\214\226.pdf" "b/lesson30-Visdom\345\217\257\350\247\206\345\214\226/30 \345\217\257\350\247\206\345\214\226.pdf" new file mode 100644 index 0000000..a11df6d Binary files /dev/null and "b/lesson30-Visdom\345\217\257\350\247\206\345\214\226/30 \345\217\257\350\247\206\345\214\226.pdf" differ diff --git "a/lesson30-Visdom\345\217\257\350\247\206\345\214\226/main.py" "b/lesson30-Visdom\345\217\257\350\247\206\345\214\226/main.py" new file mode 100644 index 0000000..798c1ea --- /dev/null +++ "b/lesson30-Visdom\345\217\257\350\247\206\345\214\226/main.py" @@ -0,0 +1,86 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms + + +batch_size=200 +learning_rate=0.01 +epochs=10 + +train_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) +test_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) + + + +class MLP(nn.Module): + + def __init__(self): + super(MLP, self).__init__() + + self.model = nn.Sequential( + nn.Linear(784, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 10), + nn.LeakyReLU(inplace=True), + ) + + def forward(self, x): + x = self.model(x) + + return x + +device = torch.device('cuda:0') +net = MLP().to(device) +optimizer = optim.SGD(net.parameters(), lr=learning_rate) +criteon = nn.CrossEntropyLoss().to(device) + +for epoch in range(epochs): + + for batch_idx, (data, target) in enumerate(train_loader): + data = data.view(-1, 28*28) + data, target = data.to(device), target.cuda() + + logits = net(data) + loss = criteon(logits, target) + + optimizer.zero_grad() + loss.backward() + # print(w1.grad.norm(), w2.grad.norm()) + optimizer.step() + + if batch_idx % 100 == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + + test_loss = 0 + correct = 0 + for data, target in test_loader: + data = data.view(-1, 28 * 28) + data, target = data.to(device), target.cuda() + logits = net(data) + test_loss += criteon(logits, target).item() + + pred = logits.argmax(dim=1) + correct += pred.eq(target).float().sum().item() + + test_loss /= len(test_loader.dataset) + print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) diff --git "a/lesson31-\350\277\207\346\213\237\345\220\210\344\270\216\346\254\240\346\213\237\345\220\210/31.pdf" "b/lesson31-\350\277\207\346\213\237\345\220\210\344\270\216\346\254\240\346\213\237\345\220\210/31.pdf" new file mode 100644 index 0000000..4be5e6b Binary files /dev/null and "b/lesson31-\350\277\207\346\213\237\345\220\210\344\270\216\346\254\240\346\213\237\345\220\210/31.pdf" differ diff --git "a/lesson32-Train-Val-Test-\344\272\244\345\217\211\351\252\214\350\257\201/32.pdf" "b/lesson32-Train-Val-Test-\344\272\244\345\217\211\351\252\214\350\257\201/32.pdf" new file mode 100644 index 0000000..bc09320 Binary files /dev/null and "b/lesson32-Train-Val-Test-\344\272\244\345\217\211\351\252\214\350\257\201/32.pdf" differ diff --git "a/lesson32-Train-Val-Test-\344\272\244\345\217\211\351\252\214\350\257\201/main.py" "b/lesson32-Train-Val-Test-\344\272\244\345\217\211\351\252\214\350\257\201/main.py" new file mode 100644 index 0000000..239ef12 --- /dev/null +++ "b/lesson32-Train-Val-Test-\344\272\244\345\217\211\351\252\214\350\257\201/main.py" @@ -0,0 +1,118 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms + + +batch_size=200 +learning_rate=0.01 +epochs=10 + +train_db = datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])) +train_loader = torch.utils.data.DataLoader( + train_db, + batch_size=batch_size, shuffle=True) + +test_db = datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) +])) +test_loader = torch.utils.data.DataLoader(test_db, + batch_size=batch_size, shuffle=True) + + +print('train:', len(train_db), 'test:', len(test_db)) +train_db, val_db = torch.utils.data.random_split(train_db, [50000, 10000]) +print('db1:', len(train_db), 'db2:', len(val_db)) +train_loader = torch.utils.data.DataLoader( + train_db, + batch_size=batch_size, shuffle=True) +val_loader = torch.utils.data.DataLoader( + val_db, + batch_size=batch_size, shuffle=True) + + + + +class MLP(nn.Module): + + def __init__(self): + super(MLP, self).__init__() + + self.model = nn.Sequential( + nn.Linear(784, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 10), + nn.LeakyReLU(inplace=True), + ) + + def forward(self, x): + x = self.model(x) + + return x + +device = torch.device('cuda:0') +net = MLP().to(device) +optimizer = optim.SGD(net.parameters(), lr=learning_rate) +criteon = nn.CrossEntropyLoss().to(device) + +for epoch in range(epochs): + + for batch_idx, (data, target) in enumerate(train_loader): + data = data.view(-1, 28*28) + data, target = data.to(device), target.cuda() + + logits = net(data) + loss = criteon(logits, target) + + optimizer.zero_grad() + loss.backward() + # print(w1.grad.norm(), w2.grad.norm()) + optimizer.step() + + if batch_idx % 100 == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + + test_loss = 0 + correct = 0 + for data, target in val_loader: + data = data.view(-1, 28 * 28) + data, target = data.to(device), target.cuda() + logits = net(data) + test_loss += criteon(logits, target).item() + + pred = logits.data.max(1)[1] + correct += pred.eq(target.data).sum() + + test_loss /= len(val_loader.dataset) + print('\nVAL set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(val_loader.dataset), + 100. * correct / len(val_loader.dataset))) + + + +test_loss = 0 +correct = 0 +for data, target in test_loader: + data = data.view(-1, 28 * 28) + data, target = data.to(device), target.cuda() + logits = net(data) + test_loss += criteon(logits, target).item() + + pred = logits.data.max(1)[1] + correct += pred.eq(target.data).sum() + +test_loss /= len(test_loader.dataset) +print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) \ No newline at end of file diff --git a/lesson33-regularization/33.pdf b/lesson33-regularization/33.pdf new file mode 100644 index 0000000..8d3c924 Binary files /dev/null and b/lesson33-regularization/33.pdf differ diff --git a/lesson33-regularization/main.py b/lesson33-regularization/main.py new file mode 100644 index 0000000..8a36592 --- /dev/null +++ b/lesson33-regularization/main.py @@ -0,0 +1,103 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms + +from visdom import Visdom + +batch_size=200 +learning_rate=0.01 +epochs=10 + +train_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + # transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) +test_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + # transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) + + + +class MLP(nn.Module): + + def __init__(self): + super(MLP, self).__init__() + + self.model = nn.Sequential( + nn.Linear(784, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 10), + nn.LeakyReLU(inplace=True), + ) + + def forward(self, x): + x = self.model(x) + + return x + +device = torch.device('cuda:0') +net = MLP().to(device) +optimizer = optim.SGD(net.parameters(), lr=learning_rate, weight_decay=0.01) +criteon = nn.CrossEntropyLoss().to(device) + +viz = Visdom() + +viz.line([0.], [0.], win='train_loss', opts=dict(title='train loss')) +viz.line([[0.0, 0.0]], [0.], win='test', opts=dict(title='test loss&acc.', + legend=['loss', 'acc.'])) +global_step = 0 + +for epoch in range(epochs): + + for batch_idx, (data, target) in enumerate(train_loader): + data = data.view(-1, 28*28) + data, target = data.to(device), target.cuda() + + logits = net(data) + loss = criteon(logits, target) + + optimizer.zero_grad() + loss.backward() + # print(w1.grad.norm(), w2.grad.norm()) + optimizer.step() + + global_step += 1 + viz.line([loss.item()], [global_step], win='train_loss', update='append') + + if batch_idx % 100 == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + + test_loss = 0 + correct = 0 + for data, target in test_loader: + data = data.view(-1, 28 * 28) + data, target = data.to(device), target.cuda() + logits = net(data) + test_loss += criteon(logits, target).item() + + pred = logits.argmax(dim=1) + correct += pred.eq(target).float().sum().item() + + viz.line([[test_loss, correct / len(test_loader.dataset)]], + [global_step], win='test', update='append') + viz.images(data.view(-1, 1, 28, 28), win='x') + viz.text(str(pred.detach().cpu().numpy()), win='pred', + opts=dict(title='pred')) + + test_loss /= len(test_loader.dataset) + print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) diff --git "a/lesson34-\345\212\250\351\207\217\344\270\216lr\350\241\260\345\207\217/34.pdf" "b/lesson34-\345\212\250\351\207\217\344\270\216lr\350\241\260\345\207\217/34.pdf" new file mode 100644 index 0000000..a2132a4 Binary files /dev/null and "b/lesson34-\345\212\250\351\207\217\344\270\216lr\350\241\260\345\207\217/34.pdf" differ diff --git a/lesson35-early stopping, dropout, sgd/35.pdf b/lesson35-early stopping, dropout, sgd/35.pdf new file mode 100644 index 0000000..0ba4547 Binary files /dev/null and b/lesson35-early stopping, dropout, sgd/35.pdf differ diff --git a/lesson35-early stopping, dropout, sgd/main.py b/lesson35-early stopping, dropout, sgd/main.py new file mode 100644 index 0000000..8043753 --- /dev/null +++ b/lesson35-early stopping, dropout, sgd/main.py @@ -0,0 +1,103 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms + +from visdom import Visdom + +batch_size=200 +learning_rate=0.01 +epochs=10 + +train_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + # transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) +test_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + # transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) + + + +class MLP(nn.Module): + + def __init__(self): + super(MLP, self).__init__() + + self.model = nn.Sequential( + nn.Linear(784, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 10), + nn.LeakyReLU(inplace=True), + ) + + def forward(self, x): + x = self.model(x) + + return x + +device = torch.device('cuda:0') +net = MLP().to(device) +optimizer = optim.SGD(net.parameters(), lr=learning_rate) +criteon = nn.CrossEntropyLoss().to(device) + +viz = Visdom() + +viz.line([0.], [0.], win='train_loss', opts=dict(title='train loss')) +viz.line([[0.0, 0.0]], [0.], win='test', opts=dict(title='test loss&acc.', + legend=['loss', 'acc.'])) +global_step = 0 + +for epoch in range(epochs): + + for batch_idx, (data, target) in enumerate(train_loader): + data = data.view(-1, 28*28) + data, target = data.to(device), target.cuda() + + logits = net(data) + loss = criteon(logits, target) + + optimizer.zero_grad() + loss.backward() + # print(w1.grad.norm(), w2.grad.norm()) + optimizer.step() + + global_step += 1 + viz.line([loss.item()], [global_step], win='train_loss', update='append') + + if batch_idx % 100 == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + + test_loss = 0 + correct = 0 + for data, target in test_loader: + data = data.view(-1, 28 * 28) + data, target = data.to(device), target.cuda() + logits = net(data) + test_loss += criteon(logits, target).item() + + pred = logits.argmax(dim=1) + correct += pred.eq(target).float().sum().item() + + viz.line([[test_loss, correct / len(test_loader.dataset)]], + [global_step], win='test', update='append') + viz.images(data.view(-1, 1, 28, 28), win='x') + viz.text(str(pred.detach().cpu().numpy()), win='pred', + opts=dict(title='pred')) + + test_loss /= len(test_loader.dataset) + print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) diff --git "a/lesson37-\344\273\200\344\271\210\346\230\257\345\215\267\347\247\257/37 \345\215\267\347\247\257.pdf" "b/lesson37-\344\273\200\344\271\210\346\230\257\345\215\267\347\247\257/37 \345\215\267\347\247\257.pdf" new file mode 100644 index 0000000..a20f044 Binary files /dev/null and "b/lesson37-\344\273\200\344\271\210\346\230\257\345\215\267\347\247\257/37 \345\215\267\347\247\257.pdf" differ diff --git "a/lesson38-\345\215\267\347\247\257\347\245\236\347\273\217\347\275\221\347\273\234/38 CNN.pdf" "b/lesson38-\345\215\267\347\247\257\347\245\236\347\273\217\347\275\221\347\273\234/38 CNN.pdf" new file mode 100644 index 0000000..58fd1ef Binary files /dev/null and "b/lesson38-\345\215\267\347\247\257\347\245\236\347\273\217\347\275\221\347\273\234/38 CNN.pdf" differ diff --git a/lesson39-Pooling&upsample/39.pdf b/lesson39-Pooling&upsample/39.pdf new file mode 100644 index 0000000..e2828d6 Binary files /dev/null and b/lesson39-Pooling&upsample/39.pdf differ diff --git "a/lesson4-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213-PyTorch\346\261\202\350\247\243/data.csv" "b/lesson4-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213-PyTorch\346\261\202\350\247\243/data.csv" new file mode 100644 index 0000000..9e8e26c --- /dev/null +++ "b/lesson4-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213-PyTorch\346\261\202\350\247\243/data.csv" @@ -0,0 +1,100 @@ +32.502345269453031,31.70700584656992 +53.426804033275019,68.77759598163891 +61.530358025636438,62.562382297945803 +47.475639634786098,71.546632233567777 +59.813207869512318,87.230925133687393 +55.142188413943821,78.211518270799232 +52.211796692214001,79.64197304980874 +39.299566694317065,59.171489321869508 +48.10504169176825,75.331242297063056 +52.550014442733818,71.300879886850353 +45.419730144973755,55.165677145959123 +54.351634881228918,82.478846757497919 +44.164049496773352,62.008923245725825 +58.16847071685779,75.392870425994957 +56.727208057096611,81.43619215887864 +48.955888566093719,60.723602440673965 +44.687196231480904,82.892503731453715 +60.297326851333466,97.379896862166078 +45.618643772955828,48.847153317355072 +38.816817537445637,56.877213186268506 +66.189816606752601,83.878564664602763 +65.41605174513407,118.59121730252249 +47.48120860786787,57.251819462268969 +41.57564261748702,51.391744079832307 +51.84518690563943,75.380651665312357 +59.370822011089523,74.765564032151374 +57.31000343834809,95.455052922574737 +63.615561251453308,95.229366017555307 +46.737619407976972,79.052406169565586 +50.556760148547767,83.432071421323712 +52.223996085553047,63.358790317497878 +35.567830047746632,41.412885303700563 +42.436476944055642,76.617341280074044 +58.16454011019286,96.769566426108199 +57.504447615341789,74.084130116602523 +45.440530725319981,66.588144414228594 +61.89622268029126,77.768482417793024 +33.093831736163963,50.719588912312084 +36.436009511386871,62.124570818071781 +37.675654860850742,60.810246649902211 +44.555608383275356,52.682983366387781 +43.318282631865721,58.569824717692867 +50.073145632289034,82.905981485070512 +43.870612645218372,61.424709804339123 +62.997480747553091,115.24415280079529 +32.669043763467187,45.570588823376085 +40.166899008703702,54.084054796223612 +53.575077531673656,87.994452758110413 +33.864214971778239,52.725494375900425 +64.707138666121296,93.576118692658241 +38.119824026822805,80.166275447370964 +44.502538064645101,65.101711570560326 +40.599538384552318,65.562301260400375 +41.720676356341293,65.280886920822823 +51.088634678336796,73.434641546324301 +55.078095904923202,71.13972785861894 +41.377726534895203,79.102829683549857 +62.494697427269791,86.520538440347153 +49.203887540826003,84.742697807826218 +41.102685187349664,59.358850248624933 +41.182016105169822,61.684037524833627 +50.186389494880601,69.847604158249183 +52.378446219236217,86.098291205774103 +50.135485486286122,59.108839267699643 +33.644706006191782,69.89968164362763 +39.557901222906828,44.862490711164398 +56.130388816875467,85.498067778840223 +57.362052133238237,95.536686846467219 +60.269214393997906,70.251934419771587 +35.678093889410732,52.721734964774988 +31.588116998132829,50.392670135079896 +53.66093226167304,63.642398775657753 +46.682228649471917,72.247251068662365 +43.107820219102464,57.812512976181402 +70.34607561504933,104.25710158543822 +44.492855880854073,86.642020318822006 +57.50453330326841,91.486778000110135 +36.930076609191808,55.231660886212836 +55.805733357942742,79.550436678507609 +38.954769073377065,44.847124242467601 +56.901214702247074,80.207523139682763 +56.868900661384046,83.14274979204346 +34.33312470421609,55.723489260543914 +59.04974121466681,77.634182511677864 +57.788223993230673,99.051414841748269 +54.282328705967409,79.120646274680027 +51.088719898979143,69.588897851118475 +50.282836348230731,69.510503311494389 +44.211741752090113,73.687564318317285 +38.005488008060688,61.366904537240131 +32.940479942618296,67.170655768995118 +53.691639571070056,85.668203145001542 +68.76573426962166,114.85387123391394 +46.230966498310252,90.123572069967423 +68.319360818255362,97.919821035242848 +50.030174340312143,81.536990783015028 +49.239765342753763,72.111832469615663 +50.039575939875988,85.232007342325673 +48.149858891028863,66.224957888054632 +25.128484647772304,53.454394214850524 diff --git "a/lesson4-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213-PyTorch\346\261\202\350\247\243/gd.py" "b/lesson4-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213-PyTorch\346\261\202\350\247\243/gd.py" new file mode 100644 index 0000000..33cb149 --- /dev/null +++ "b/lesson4-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213-PyTorch\346\261\202\350\247\243/gd.py" @@ -0,0 +1,50 @@ +import numpy as np + +# y = wx + b +def compute_error_for_line_given_points(b, w, points): + totalError = 0 + for i in range(0, len(points)): + x = points[i, 0] + y = points[i, 1] + totalError += (y - (w * x + b)) ** 2 + return totalError / float(len(points)) + +def step_gradient(b_current, w_current, points, learningRate): + b_gradient = 0 + w_gradient = 0 + N = float(len(points)) + for i in range(0, len(points)): + x = points[i, 0] + y = points[i, 1] + b_gradient += -(2/N) * (y - ((w_current * x) + b_current)) + w_gradient += -(2/N) * x * (y - ((w_current * x) + b_current)) + new_b = b_current - (learningRate * b_gradient) + new_m = w_current - (learningRate * w_gradient) + return [new_b, new_m] + +def gradient_descent_runner(points, starting_b, starting_m, learning_rate, num_iterations): + b = starting_b + m = starting_m + for i in range(num_iterations): + b, m = step_gradient(b, m, np.array(points), learning_rate) + return [b, m] + +def run(): + points = np.genfromtxt("data.csv", delimiter=",") + learning_rate = 0.0001 + initial_b = 0 # initial y-intercept guess + initial_m = 0 # initial slope guess + num_iterations = 1000 + print("Starting gradient descent at b = {0}, m = {1}, error = {2}" + .format(initial_b, initial_m, + compute_error_for_line_given_points(initial_b, initial_m, points)) + ) + print("Running...") + [b, m] = gradient_descent_runner(points, initial_b, initial_m, learning_rate, num_iterations) + print("After {0} iterations b = {1}, m = {2}, error = {3}". + format(num_iterations, b, m, + compute_error_for_line_given_points(b, m, points)) + ) + +if __name__ == '__main__': + run() \ No newline at end of file diff --git "a/lesson4-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213-PyTorch\346\261\202\350\247\243/lesson4.pdf" "b/lesson4-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213-PyTorch\346\261\202\350\247\243/lesson4.pdf" new file mode 100644 index 0000000..4659e93 Binary files /dev/null and "b/lesson4-\347\256\200\345\215\225\345\233\236\345\275\222\346\241\210\344\276\213-PyTorch\346\261\202\350\247\243/lesson4.pdf" differ diff --git a/lesson40-BatchNorm/40.pdf b/lesson40-BatchNorm/40.pdf new file mode 100644 index 0000000..4a88d05 Binary files /dev/null and b/lesson40-BatchNorm/40.pdf differ diff --git a/lesson41-LeNet5,AlexNet, VGG, GoogLeNet/41.pdf b/lesson41-LeNet5,AlexNet, VGG, GoogLeNet/41.pdf new file mode 100644 index 0000000..25e933b Binary files /dev/null and b/lesson41-LeNet5,AlexNet, VGG, GoogLeNet/41.pdf differ diff --git a/lesson42-ResNet/42.pdf b/lesson42-ResNet/42.pdf new file mode 100644 index 0000000..5c4f976 Binary files /dev/null and b/lesson42-ResNet/42.pdf differ diff --git a/lesson42-ResNet/resnet.py b/lesson42-ResNet/resnet.py new file mode 100644 index 0000000..4b8dcbe --- /dev/null +++ b/lesson42-ResNet/resnet.py @@ -0,0 +1,173 @@ +import torch +from torch import nn +from torch.nn import functional as F +from torch.utils.data import DataLoader +from torchvision import datasets +from torchvision import transforms +from torch import nn, optim + +# from torchvision.models import resnet18 + +class ResBlk(nn.Module): + """ + resnet block + """ + + def __init__(self, ch_in, ch_out): + """ + :param ch_in: + :param ch_out: + """ + super(ResBlk, self).__init__() + + self.conv1 = nn.Conv2d(ch_in, ch_out, kernel_size=3, stride=1, padding=1) + self.bn1 = nn.BatchNorm2d(ch_out) + self.conv2 = nn.Conv2d(ch_out, ch_out, kernel_size=3, stride=1, padding=1) + self.bn2 = nn.BatchNorm2d(ch_out) + + self.extra = nn.Sequential() + if ch_out != ch_in: + # [b, ch_in, h, w] => [b, ch_out, h, w] + self.extra = nn.Sequential( + nn.Conv2d(ch_in, ch_out, kernel_size=1, stride=1), + nn.BatchNorm2d(ch_out) + ) + + + def forward(self, x): + """ + :param x: [b, ch, h, w] + :return: + """ + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + # short cut. + # extra module: [b, ch_in, h, w] => [b, ch_out, h, w] + # element-wise add: + out = self.extra(x) + out + + return out + + + + +class ResNet18(nn.Module): + + def __init__(self): + super(ResNet18, self).__init__() + + self.conv1 = nn.Sequential( + nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(16) + ) + # followed 4 blocks + # [b, 64, h, w] => [b, 128, h ,w] + self.blk1 = ResBlk(16, 16) + # [b, 128, h, w] => [b, 256, h, w] + self.blk2 = ResBlk(16, 32) + # # [b, 256, h, w] => [b, 512, h, w] + # self.blk3 = ResBlk(128, 256) + # # [b, 512, h, w] => [b, 1024, h, w] + # self.blk4 = ResBlk(256, 512) + + self.outlayer = nn.Linear(32*32*32, 10) + + def forward(self, x): + """ + :param x: + :return: + """ + x = F.relu(self.conv1(x)) + + # [b, 64, h, w] => [b, 1024, h, w] + x = self.blk1(x) + x = self.blk2(x) + # x = self.blk3(x) + # x = self.blk4(x) + + # print(x.shape) + x = x.view(x.size(0), -1) + x = self.outlayer(x) + + + return x + + + +def main(): + batchsz = 32 + + cifar_train = datasets.CIFAR10('cifar', True, transform=transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor() + ]), download=True) + cifar_train = DataLoader(cifar_train, batch_size=batchsz, shuffle=True) + + cifar_test = datasets.CIFAR10('cifar', False, transform=transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor() + ]), download=True) + cifar_test = DataLoader(cifar_test, batch_size=batchsz, shuffle=True) + + + x, label = iter(cifar_train).next() + print('x:', x.shape, 'label:', label.shape) + + device = torch.device('cuda') + # model = Lenet5().to(device) + model = ResNet18().to(device) + + criteon = nn.CrossEntropyLoss().to(device) + optimizer = optim.Adam(model.parameters(), lr=1e-3) + print(model) + + for epoch in range(1000): + + model.train() + for batchidx, (x, label) in enumerate(cifar_train): + # [b, 3, 32, 32] + # [b] + x, label = x.to(device), label.to(device) + + logits = model(x) + # logits: [b, 10] + # label: [b] + # loss: tensor scalar + loss = criteon(logits, label) + + # backprop + optimizer.zero_grad() + loss.backward() + optimizer.step() + + + # + print(epoch, 'loss:', loss.item()) + + + model.eval() + with torch.no_grad(): + # test + total_correct = 0 + total_num = 0 + for x, label in cifar_test: + # [b, 3, 32, 32] + # [b] + x, label = x.to(device), label.to(device) + + # [b, 10] + logits = model(x) + # [b] + pred = logits.argmax(dim=1) + # [b] vs [b] => scalar tensor + correct = torch.eq(pred, label).float().sum().item() + total_correct += correct + total_num += x.size(0) + # print(correct) + + acc = total_correct / total_num + print(epoch, 'acc:', acc) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/lesson43-nn.Module/43.pdf b/lesson43-nn.Module/43.pdf new file mode 100644 index 0000000..6eb61f5 Binary files /dev/null and b/lesson43-nn.Module/43.pdf differ diff --git a/lesson43-nn.Module/main.py b/lesson43-nn.Module/main.py new file mode 100644 index 0000000..1a8acf6 --- /dev/null +++ b/lesson43-nn.Module/main.py @@ -0,0 +1,105 @@ +import torch +from torch import nn +from torch import optim + + + +class MyLinear(nn.Module): + + def __init__(self, inp, outp): + super(MyLinear, self).__init__() + + # requires_grad = True + self.w = nn.Parameter(torch.randn(outp, inp)) + self.b = nn.Parameter(torch.randn(outp)) + + def forward(self, x): + x = x @ self.w.t() + self.b + return x + + +class Flatten(nn.Module): + + def __init__(self): + super(Flatten, self).__init__() + + def forward(self, input): + return input.view(input.size(0), -1) + + + +class TestNet(nn.Module): + + def __init__(self): + super(TestNet, self).__init__() + + self.net = nn.Sequential(nn.Conv2d(1, 16, stride=1, padding=1), + nn.MaxPool2d(2, 2), + Flatten(), + nn.Linear(1*14*14, 10)) + + def forward(self, x): + return self.net(x) + + +class BasicNet(nn.Module): + + def __init__(self): + super(BasicNet, self).__init__() + + self.net = nn.Linear(4, 3) + + def forward(self, x): + return self.net(x) + + + +class Net(nn.Module): + + def __init__(self): + super(Net, self).__init__() + + self.net = nn.Sequential(BasicNet(), + nn.ReLU(), + nn.Linear(3, 2)) + + def forward(self, x): + return self.net(x) + + + + + +def main(): + device = torch.device('cuda') + net = Net() + net.to(device) + + net.train() + + net.eval() + + # net.load_state_dict(torch.load('ckpt.mdl')) + # + # + # torch.save(net.state_dict(), 'ckpt.mdl') + + for name, t in net.named_parameters(): + print('parameters:', name, t.shape) + + for name, m in net.named_children(): + print('children:', name, m) + + + for name, m in net.named_modules(): + print('modules:', name, m) + + + + + + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git "a/lesson44-\346\225\260\346\215\256\345\242\236\345\274\272/44.pdf" "b/lesson44-\346\225\260\346\215\256\345\242\236\345\274\272/44.pdf" new file mode 100644 index 0000000..e36f77b Binary files /dev/null and "b/lesson44-\346\225\260\346\215\256\345\242\236\345\274\272/44.pdf" differ diff --git "a/lesson44-\346\225\260\346\215\256\345\242\236\345\274\272/main.py" "b/lesson44-\346\225\260\346\215\256\345\242\236\345\274\272/main.py" new file mode 100644 index 0000000..c1c1480 --- /dev/null +++ "b/lesson44-\346\225\260\346\215\256\345\242\236\345\274\272/main.py" @@ -0,0 +1,109 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms + +from visdom import Visdom + +batch_size=200 +learning_rate=0.01 +epochs=10 + +train_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.RandomHorizontalFlip(), + transforms.RandomVerticalFlip(), + transforms.RandomRotation(15), + transforms.RandomRotation([90, 180, 270]), + transforms.Resize([32, 32]), + transforms.RandomCrop([28, 28]), + transforms.ToTensor(), + # transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) +test_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + # transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=batch_size, shuffle=True) + + + +class MLP(nn.Module): + + def __init__(self): + super(MLP, self).__init__() + + self.model = nn.Sequential( + nn.Linear(784, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 200), + nn.LeakyReLU(inplace=True), + nn.Linear(200, 10), + nn.LeakyReLU(inplace=True), + ) + + def forward(self, x): + x = self.model(x) + + return x + +device = torch.device('cuda:0') +net = MLP().to(device) +optimizer = optim.SGD(net.parameters(), lr=learning_rate) +criteon = nn.CrossEntropyLoss().to(device) + +viz = Visdom() + +viz.line([0.], [0.], win='train_loss', opts=dict(title='train loss')) +viz.line([[0.0, 0.0]], [0.], win='test', opts=dict(title='test loss&acc.', + legend=['loss', 'acc.'])) +global_step = 0 + +for epoch in range(epochs): + + for batch_idx, (data, target) in enumerate(train_loader): + data = data.view(-1, 28*28) + data, target = data.to(device), target.cuda() + + logits = net(data) + loss = criteon(logits, target) + + optimizer.zero_grad() + loss.backward() + # print(w1.grad.norm(), w2.grad.norm()) + optimizer.step() + + global_step += 1 + viz.line([loss.item()], [global_step], win='train_loss', update='append') + + if batch_idx % 100 == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + + test_loss = 0 + correct = 0 + for data, target in test_loader: + data = data.view(-1, 28 * 28) + data, target = data.to(device), target.cuda() + logits = net(data) + test_loss += criteon(logits, target).item() + + pred = logits.argmax(dim=1) + correct += pred.eq(target).float().sum().item() + + viz.line([[test_loss, correct / len(test_loader.dataset)]], + [global_step], win='test', update='append') + viz.images(data.view(-1, 1, 28, 28), win='x') + viz.text(str(pred.detach().cpu().numpy()), win='pred', + opts=dict(title='pred')) + + test_loss /= len(test_loader.dataset) + print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) diff --git "a/lesson46-\346\227\266\351\227\264\345\272\217\345\210\227\350\241\250\347\244\272/46.pdf" "b/lesson46-\346\227\266\351\227\264\345\272\217\345\210\227\350\241\250\347\244\272/46.pdf" new file mode 100644 index 0000000..a57ed44 Binary files /dev/null and "b/lesson46-\346\227\266\351\227\264\345\272\217\345\210\227\350\241\250\347\244\272/46.pdf" differ diff --git "a/lesson46-\346\227\266\351\227\264\345\272\217\345\210\227\350\241\250\347\244\272/main.py" "b/lesson46-\346\227\266\351\227\264\345\272\217\345\210\227\350\241\250\347\244\272/main.py" new file mode 100644 index 0000000..dd495f7 --- /dev/null +++ "b/lesson46-\346\227\266\351\227\264\345\272\217\345\210\227\350\241\250\347\244\272/main.py" @@ -0,0 +1,19 @@ +import torch +import torchnlp + +from torchnlp import word_to_vector + + + + + + +def main(): + + # vec = word_to_vector.GloVe() + vec = word_to_vector.BPEmb() + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git "a/lesson47-RNN\345\216\237\347\220\206/47.pdf" "b/lesson47-RNN\345\216\237\347\220\206/47.pdf" new file mode 100644 index 0000000..6ebc1d2 Binary files /dev/null and "b/lesson47-RNN\345\216\237\347\220\206/47.pdf" differ diff --git "a/lesson48-RNN Layer\344\275\277\347\224\250/48.pdf" "b/lesson48-RNN Layer\344\275\277\347\224\250/48.pdf" new file mode 100644 index 0000000..7848b8b Binary files /dev/null and "b/lesson48-RNN Layer\344\275\277\347\224\250/48.pdf" differ diff --git "a/lesson48-RNN Layer\344\275\277\347\224\250/rnn.py" "b/lesson48-RNN Layer\344\275\277\347\224\250/rnn.py" new file mode 100644 index 0000000..c326002 --- /dev/null +++ "b/lesson48-RNN Layer\344\275\277\347\224\250/rnn.py" @@ -0,0 +1,79 @@ +import torch +from torch import nn +from torch import optim +from torch.nn import functional as F + + + + + + +def main(): + + + rnn = nn.RNN(input_size=100, hidden_size=20, num_layers=1) + print(rnn) + x = torch.randn(10, 3, 100) + out, h = rnn(x, torch.zeros(1, 3, 20)) + print(out.shape, h.shape) + + rnn = nn.RNN(input_size=100, hidden_size=20, num_layers=4) + print(rnn) + x = torch.randn(10, 3, 100) + out, h = rnn(x, torch.zeros(4, 3, 20)) + print(out.shape, h.shape) + # print(vars(rnn)) + + print('rnn by cell') + + cell1 = nn.RNNCell(100, 20) + h1 = torch.zeros(3, 20) + for xt in x: + h1 = cell1(xt, h1) + print(h1.shape) + + + cell1 = nn.RNNCell(100, 30) + cell2 = nn.RNNCell(30, 20) + h1 = torch.zeros(3, 30) + h2 = torch.zeros(3, 20) + for xt in x: + h1 = cell1(xt, h1) + h2 = cell2(h1, h2) + print(h2.shape) + + print('Lstm') + lstm = nn.LSTM(input_size=100, hidden_size=20, num_layers=4) + print(lstm) + x = torch.randn(10, 3, 100) + out, (h, c) = lstm(x) + print(out.shape, h.shape, c.shape) + + print('one layer lstm') + cell = nn.LSTMCell(input_size=100, hidden_size=20) + h = torch.zeros(3, 20) + c = torch.zeros(3, 20) + for xt in x: + h, c = cell(xt, [h, c]) + print(h.shape, c.shape) + + + print('two layer lstm') + cell1 = nn.LSTMCell(input_size=100, hidden_size=30) + cell2 = nn.LSTMCell(input_size=30, hidden_size=20) + h1 = torch.zeros(3, 30) + c1 = torch.zeros(3, 30) + h2 = torch.zeros(3, 20) + c2 = torch.zeros(3, 20) + for xt in x: + h1, c1 = cell1(xt, [h1, c1]) + h2, c2 = cell2(h1, [h2, c2]) + print(h2.shape, c2.shape) + + + + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git "a/lesson49-\346\227\266\351\227\264\345\272\217\345\210\227\351\242\204\346\265\213/49.pdf" "b/lesson49-\346\227\266\351\227\264\345\272\217\345\210\227\351\242\204\346\265\213/49.pdf" new file mode 100644 index 0000000..7e31d4f Binary files /dev/null and "b/lesson49-\346\227\266\351\227\264\345\272\217\345\210\227\351\242\204\346\265\213/49.pdf" differ diff --git "a/lesson49-\346\227\266\351\227\264\345\272\217\345\210\227\351\242\204\346\265\213/seris.py" "b/lesson49-\346\227\266\351\227\264\345\272\217\345\210\227\351\242\204\346\265\213/seris.py" new file mode 100644 index 0000000..9d2eb78 --- /dev/null +++ "b/lesson49-\346\227\266\351\227\264\345\272\217\345\210\227\351\242\204\346\265\213/seris.py" @@ -0,0 +1,93 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim +from matplotlib import pyplot as plt + + +num_time_steps = 50 +input_size = 1 +hidden_size = 16 +output_size = 1 +lr=0.01 + + + +class Net(nn.Module): + + def __init__(self, ): + super(Net, self).__init__() + + self.rnn = nn.RNN( + input_size=input_size, + hidden_size=hidden_size, + num_layers=1, + batch_first=True, + ) + for p in self.rnn.parameters(): + nn.init.normal_(p, mean=0.0, std=0.001) + + self.linear = nn.Linear(hidden_size, output_size) + + def forward(self, x, hidden_prev): + + out, hidden_prev = self.rnn(x, hidden_prev) + # [b, seq, h] + out = out.view(-1, hidden_size) + out = self.linear(out) + out = out.unsqueeze(dim=0) + return out, hidden_prev + + + + +model = Net() +criterion = nn.MSELoss() +optimizer = optim.Adam(model.parameters(), lr) + +hidden_prev = torch.zeros(1, 1, hidden_size) + +for iter in range(6000): + start = np.random.randint(3, size=1)[0] + time_steps = np.linspace(start, start + 10, num_time_steps) + data = np.sin(time_steps) + data = data.reshape(num_time_steps, 1) + x = torch.tensor(data[:-1]).float().view(1, num_time_steps - 1, 1) + y = torch.tensor(data[1:]).float().view(1, num_time_steps - 1, 1) + + output, hidden_prev = model(x, hidden_prev) + hidden_prev = hidden_prev.detach() + + loss = criterion(output, y) + model.zero_grad() + loss.backward() + # for p in model.parameters(): + # print(p.grad.norm()) + # torch.nn.utils.clip_grad_norm_(p, 10) + optimizer.step() + + if iter % 100 == 0: + print("Iteration: {} loss {}".format(iter, loss.item())) + +start = np.random.randint(3, size=1)[0] +time_steps = np.linspace(start, start + 10, num_time_steps) +data = np.sin(time_steps) +data = data.reshape(num_time_steps, 1) +x = torch.tensor(data[:-1]).float().view(1, num_time_steps - 1, 1) +y = torch.tensor(data[1:]).float().view(1, num_time_steps - 1, 1) + +predictions = [] +input = x[:, 0, :] +for _ in range(x.shape[1]): + input = input.view(1, 1, 1) + (pred, hidden_prev) = model(input, hidden_prev) + input = pred + predictions.append(pred.detach().numpy().ravel()[0]) + +x = x.data.numpy().ravel() +y = y.data.numpy() +plt.scatter(time_steps[:-1], x.ravel(), s=90) +plt.plot(time_steps[:-1], x.ravel()) + +plt.scatter(time_steps[1:], predictions) +plt.show() \ No newline at end of file diff --git "a/lesson5-\346\211\213\345\206\231\346\225\260\345\255\227\351\227\256\351\242\230\345\274\225\345\205\245/lesson5.pdf" "b/lesson5-\346\211\213\345\206\231\346\225\260\345\255\227\351\227\256\351\242\230\345\274\225\345\205\245/lesson5.pdf" new file mode 100644 index 0000000..e502902 Binary files /dev/null and "b/lesson5-\346\211\213\345\206\231\346\225\260\345\255\227\351\227\256\351\242\230\345\274\225\345\205\245/lesson5.pdf" differ diff --git "a/lesson50-RNN\350\256\255\347\273\203\351\232\276\351\242\230/50.pdf" "b/lesson50-RNN\350\256\255\347\273\203\351\232\276\351\242\230/50.pdf" new file mode 100644 index 0000000..63eb60e Binary files /dev/null and "b/lesson50-RNN\350\256\255\347\273\203\351\232\276\351\242\230/50.pdf" differ diff --git "a/lesson51-LSTM\345\216\237\347\220\206/51.pdf" "b/lesson51-LSTM\345\216\237\347\220\206/51.pdf" new file mode 100644 index 0000000..511f6d5 Binary files /dev/null and "b/lesson51-LSTM\345\216\237\347\220\206/51.pdf" differ diff --git "a/lesson52-LSTM Layer\344\275\277\347\224\250/52.pdf" "b/lesson52-LSTM Layer\344\275\277\347\224\250/52.pdf" new file mode 100644 index 0000000..4b91091 Binary files /dev/null and "b/lesson52-LSTM Layer\344\275\277\347\224\250/52.pdf" differ diff --git "a/lesson53-\346\203\205\346\204\237\345\210\206\347\261\273\345\256\236\346\210\230/53.pdf" "b/lesson53-\346\203\205\346\204\237\345\210\206\347\261\273\345\256\236\346\210\230/53.pdf" new file mode 100644 index 0000000..8657a8b Binary files /dev/null and "b/lesson53-\346\203\205\346\204\237\345\210\206\347\261\273\345\256\236\346\210\230/53.pdf" differ diff --git "a/lesson53-\346\203\205\346\204\237\345\210\206\347\261\273\345\256\236\346\210\230/lstm.ipynb" "b/lesson53-\346\203\205\346\204\237\345\210\206\347\261\273\345\256\236\346\210\230/lstm.ipynb" new file mode 100644 index 0000000..a13dbb6 --- /dev/null +++ "b/lesson53-\346\203\205\346\204\237\345\210\206\347\261\273\345\256\236\346\210\230/lstm.ipynb" @@ -0,0 +1,1240 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "lstm", + "version": "0.3.2", + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "metadata": { + "id": "xazrh9eIcgTO", + "colab_type": "code", + "outputId": "eed0e41e-8fad-4940-bab3-8e0a147e0871", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 383 + } + }, + "cell_type": "code", + "source": [ + "!pip install torch\n", + "!pip install torchtext\n", + "!python -m spacy download en\n", + "\n", + "\n", + "# K80 gpu for 12 hours\n", + "import torch\n", + "from torch import nn, optim\n", + "from torchtext import data, datasets\n", + "\n", + "print('GPU:', torch.cuda.is_available())\n", + "\n", + "torch.manual_seed(123)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (0.4.1)\n", + "Requirement already satisfied: torchtext in /usr/local/lib/python3.6/dist-packages (0.3.1)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchtext) (1.14.6)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from torchtext) (4.28.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from torchtext) (2.18.4)\n", + "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from torchtext) (0.4.1)\n", + "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (1.22)\n", + "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (3.0.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2018.11.29)\n", + "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2.6)\n", + "Requirement already satisfied: en_core_web_sm==2.0.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz#egg=en_core_web_sm==2.0.0 in /usr/local/lib/python3.6/dist-packages (2.0.0)\n", + "\n", + "\u001b[93m Linking successful\u001b[0m\n", + " /usr/local/lib/python3.6/dist-packages/en_core_web_sm -->\n", + " /usr/local/lib/python3.6/dist-packages/spacy/data/en\n", + "\n", + " You can now load the model via spacy.load('en')\n", + "\n", + "GPU: True\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 1 + } + ] + }, + { + "metadata": { + "id": "sPOkbQz1dfMS", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "\n", + "\n", + "TEXT = data.Field(tokenize='spacy')\n", + "LABEL = data.LabelField(dtype=torch.float)\n", + "train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "LodNOFuEeRuv", + "colab_type": "code", + "outputId": "742cd4b9-10a2-4de8-f0e3-2c525b3d5364", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + } + }, + "cell_type": "code", + "source": [ + "print('len of train data:', len(train_data))\n", + "print('len of test data:', len(test_data))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "len of train data: 25000\n", + "len of test data: 25000\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "gnQaJuCLee2o", + "colab_type": "code", + "outputId": "fb9424f5-2604-4680-c472-2557c0988817", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 72 + } + }, + "cell_type": "code", + "source": [ + "print(train_data.examples[15].text)\n", + "print(train_data.examples[15].label)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "['Well', 'when', 'watching', 'this', 'film', 'late', 'one', 'night', 'I', 'was', 'simple', 'amazed', 'by', 'it', \"'s\", 'greatness', '.', 'Fantastic', 'script', ',', 'great', 'acting', ',', 'costumes', 'and', 'special', 'effects', ',', 'and', 'the', 'plot', 'twists', ',', 'wow', '!', '!', 'In', 'fact', 'if', 'you', 'can', 'see', 'the', 'ending', 'coming', 'you', 'should', 'become', 'a', 'writer', 'yourself.Great', ',', 'I', 'would', 'recommend', 'this', 'film', 'to', 'anyone', ',', 'especially', 'if', 'I', 'don;t', 'like', 'them', 'much.Terrific']\n", + "pos\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "u3R5sgSme-Tt", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# word2vec, glove\n", + "TEXT.build_vocab(train_data, max_size=10000, vectors='glove.6B.100d')\n", + "LABEL.build_vocab(train_data)\n", + "\n", + "\n", + "batchsz = 30\n", + "device = torch.device('cuda')\n", + "train_iterator, test_iterator = data.BucketIterator.splits(\n", + " (train_data, test_data),\n", + " batch_size = batchsz,\n", + " device=device\n", + ")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "PBKKxxFBgRTM", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "class RNN(nn.Module):\n", + " \n", + " def __init__(self, vocab_size, embedding_dim, hidden_dim):\n", + " \"\"\"\n", + " \"\"\"\n", + " super(RNN, self).__init__()\n", + " \n", + " # [0-10001] => [100]\n", + " self.embedding = nn.Embedding(vocab_size, embedding_dim)\n", + " # [100] => [256]\n", + " self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers=2, \n", + " bidirectional=True, dropout=0.5)\n", + " # [256*2] => [1]\n", + " self.fc = nn.Linear(hidden_dim*2, 1)\n", + " self.dropout = nn.Dropout(0.5)\n", + " \n", + " \n", + " def forward(self, x):\n", + " \"\"\"\n", + " x: [seq_len, b] vs [b, 3, 28, 28]\n", + " \"\"\"\n", + " # [seq, b, 1] => [seq, b, 100]\n", + " embedding = self.dropout(self.embedding(x))\n", + " \n", + " # output: [seq, b, hid_dim*2]\n", + " # hidden/h: [num_layers*2, b, hid_dim]\n", + " # cell/c: [num_layers*2, b, hid_di]\n", + " output, (hidden, cell) = self.rnn(embedding)\n", + " \n", + " # [num_layers*2, b, hid_dim] => 2 of [b, hid_dim] => [b, hid_dim*2]\n", + " hidden = torch.cat([hidden[-2], hidden[-1]], dim=1)\n", + " \n", + " # [b, hid_dim*2] => [b, 1]\n", + " hidden = self.dropout(hidden)\n", + " out = self.fc(hidden)\n", + " \n", + " return out" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "cxq70oc9lK-4", + "colab_type": "code", + "outputId": "b7bb2f08-0c99-4ac1-9594-e5e98168c19d", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 155 + } + }, + "cell_type": "code", + "source": [ + "rnn = RNN(len(TEXT.vocab), 100, 256)\n", + "\n", + "pretrained_embedding = TEXT.vocab.vectors\n", + "print('pretrained_embedding:', pretrained_embedding.shape)\n", + "rnn.embedding.weight.data.copy_(pretrained_embedding)\n", + "print('embedding layer inited.')\n", + "\n", + "optimizer = optim.Adam(rnn.parameters(), lr=1e-3)\n", + "criteon = nn.BCEWithLogitsLoss().to(device)\n", + "rnn.to(device)\n" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "pretrained_embedding: torch.Size([10002, 100])\n", + "embedding layer inited.\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "RNN(\n", + " (embedding): Embedding(10002, 100)\n", + " (rnn): LSTM(100, 256, num_layers=2, dropout=0.5, bidirectional=True)\n", + " (fc): Linear(in_features=512, out_features=1, bias=True)\n", + " (dropout): Dropout(p=0.5)\n", + ")" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 7 + } + ] + }, + { + "metadata": { + "id": "_Rw_PZsZnBuJ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import numpy as np\n", + "\n", + "def binary_acc(preds, y):\n", + " \"\"\"\n", + " get accuracy\n", + " \"\"\"\n", + " preds = torch.round(torch.sigmoid(preds))\n", + " correct = torch.eq(preds, y).float()\n", + " acc = correct.sum() / len(correct)\n", + " return acc\n", + "\n", + "def train(rnn, iterator, optimizer, criteon):\n", + " \n", + " avg_acc = []\n", + " rnn.train()\n", + " \n", + " for i, batch in enumerate(iterator):\n", + " \n", + " # [seq, b] => [b, 1] => [b]\n", + " pred = rnn(batch.text).squeeze(1)\n", + " # \n", + " loss = criteon(pred, batch.label)\n", + " acc = binary_acc(pred, batch.label).item()\n", + " avg_acc.append(acc)\n", + " \n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + " if i%10 == 0:\n", + " print(i, acc)\n", + " \n", + " avg_acc = np.array(avg_acc).mean()\n", + " print('avg acc:', avg_acc)\n", + " \n", + " \n", + "def eval(rnn, iterator, criteon):\n", + " \n", + " avg_acc = []\n", + " \n", + " rnn.eval()\n", + " \n", + " with torch.no_grad():\n", + " for batch in iterator:\n", + "\n", + " # [b, 1] => [b]\n", + " pred = rnn(batch.text).squeeze(1)\n", + "\n", + " #\n", + " loss = criteon(pred, batch.label)\n", + "\n", + " acc = binary_acc(pred, batch.label).item()\n", + " avg_acc.append(acc)\n", + " \n", + " avg_acc = np.array(avg_acc).mean()\n", + " \n", + " print('>>test:', avg_acc)\n", + " \n", + " \n", + " " + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "lrjzCiiao4Qw", + "colab_type": "code", + "outputId": "ddc45f41-982d-4afc-e1c1-36e96a0a6e76", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 14878 + } + }, + "cell_type": "code", + "source": [ + "for epoch in range(10):\n", + " \n", + " eval(rnn, test_iterator, criteon)\n", + " train(rnn, train_iterator, optimizer, criteon)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + ">>test: 0.4997602136050769\n", + "0 0.46666669845581055\n", + "10 0.40000003576278687\n", + "20 0.5\n", + "30 0.5\n", + "40 0.4333333671092987\n", + "50 0.5333333611488342\n", + "60 0.6000000238418579\n", + "70 0.5666667222976685\n", + "80 0.40000003576278687\n", + "90 0.36666667461395264\n", + "100 0.5333333611488342\n", + "110 0.6666666865348816\n", + "120 0.7333333492279053\n", + "130 0.4333333671092987\n", + "140 0.6000000238418579\n", + "150 0.5333333611488342\n", + "160 0.5\n", + "170 0.46666669845581055\n", + "180 0.6333333849906921\n", + "190 0.6666666865348816\n", + "200 0.40000003576278687\n", + "210 0.46666669845581055\n", + "220 0.5666667222976685\n", + "230 0.5\n", + "240 0.7333333492279053\n", + "250 0.6666666865348816\n", + "260 0.6333333849906921\n", + "270 0.6666666865348816\n", + "280 0.6000000238418579\n", + "290 0.7666667103767395\n", + "300 0.6000000238418579\n", + "310 0.5666667222976685\n", + "320 0.8333333730697632\n", + "330 0.6333333849906921\n", + "340 0.6000000238418579\n", + "350 0.6333333849906921\n", + "360 0.6333333849906921\n", + "370 0.7000000476837158\n", + "380 0.7333333492279053\n", + "390 0.6000000238418579\n", + "400 0.7666667103767395\n", + "410 0.7333333492279053\n", + "420 0.8000000715255737\n", + "430 0.6666666865348816\n", + "440 0.7666667103767395\n", + "450 0.5\n", + "460 0.46666669845581055\n", + "470 0.7000000476837158\n", + "480 0.5333333611488342\n", + "490 0.36666667461395264\n", + "500 0.7000000476837158\n", + "510 0.7000000476837158\n", + "520 0.5333333611488342\n", + "530 0.6666666865348816\n", + "540 0.40000003576278687\n", + "550 0.5333333611488342\n", + "560 0.5666667222976685\n", + "570 0.6666666865348816\n", + "580 0.6000000238418579\n", + "590 0.5\n", + "600 0.6000000238418579\n", + "610 0.5333333611488342\n", + "620 0.5666667222976685\n", + "630 0.5666667222976685\n", + "640 0.6000000238418579\n", + "650 0.5333333611488342\n", + "660 0.6666666865348816\n", + "670 0.6000000238418579\n", + "680 0.5333333611488342\n", + "690 0.4333333671092987\n", + "700 0.46666669845581055\n", + "710 0.6333333849906921\n", + "720 0.5333333611488342\n", + "730 0.6000000238418579\n", + "740 0.6000000238418579\n", + "750 0.7333333492279053\n", + "760 0.40000003576278687\n", + "770 0.8666667342185974\n", + "780 0.40000003576278687\n", + "790 0.5\n", + "800 0.5333333611488342\n", + "810 0.5\n", + "820 0.5666667222976685\n", + "830 0.6333333849906921\n", + "avg acc: 0.5920064268852595\n", + ">>test: 0.5261790834688883\n", + "0 0.6000000238418579\n", + "10 0.6333333849906921\n", + "20 0.7000000476837158\n", + "30 0.6666666865348816\n", + "40 0.7000000476837158\n", + "50 0.7000000476837158\n", + "60 0.5333333611488342\n", + "70 0.7000000476837158\n", + "80 0.7000000476837158\n", + "90 0.7333333492279053\n", + "100 0.7666667103767395\n", + "110 0.7000000476837158\n", + "120 0.9000000357627869\n", + "130 0.8000000715255737\n", + "140 0.8333333730697632\n", + "150 0.6333333849906921\n", + "160 0.7000000476837158\n", + "170 0.9333333969116211\n", + "180 0.8333333730697632\n", + "190 0.7666667103767395\n", + "200 0.8666667342185974\n", + "210 0.8333333730697632\n", + "220 0.8666667342185974\n", + "230 0.8000000715255737\n", + "240 0.7000000476837158\n", + "250 0.9000000357627869\n", + "260 0.8000000715255737\n", + "270 0.8666667342185974\n", + "280 0.8333333730697632\n", + "290 0.8333333730697632\n", + "300 0.7000000476837158\n", + "310 0.8666667342185974\n", + "320 0.8000000715255737\n", + "330 0.8000000715255737\n", + "340 0.7666667103767395\n", + "350 0.9333333969116211\n", + "360 0.8666667342185974\n", + "370 0.8000000715255737\n", + "380 0.8333333730697632\n", + "390 0.9000000357627869\n", + "400 0.8666667342185974\n", + "410 0.7666667103767395\n", + "420 0.7666667103767395\n", + "430 0.6666666865348816\n", + "440 0.9000000357627869\n", + "450 0.8333333730697632\n", + "460 0.9000000357627869\n", + "470 0.8333333730697632\n", + "480 0.7666667103767395\n", + "490 0.9000000357627869\n", + "500 0.9000000357627869\n", + "510 0.8666667342185974\n", + "520 0.9000000357627869\n", + "530 0.8666667342185974\n", + "540 0.7333333492279053\n", + "550 0.9000000357627869\n", + "560 0.9333333969116211\n", + "570 0.9000000357627869\n", + "580 0.8000000715255737\n", + "590 0.8333333730697632\n", + "600 0.8666667342185974\n", + "610 0.8333333730697632\n", + "620 0.8000000715255737\n", + "630 0.8666667342185974\n", + "640 0.9666666984558105\n", + "650 0.8666667342185974\n", + "660 0.8000000715255737\n", + "670 0.9000000357627869\n", + "680 0.8333333730697632\n", + "690 0.9000000357627869\n", + "700 0.8333333730697632\n", + "710 0.8333333730697632\n", + "720 0.9000000357627869\n", + "730 0.8333333730697632\n", + "740 0.8000000715255737\n", + "750 0.9333333969116211\n", + "760 0.8000000715255737\n", + "770 0.8666667342185974\n", + "780 0.8666667342185974\n", + "790 0.8000000715255737\n", + "800 0.8666667342185974\n", + "810 0.8333333730697632\n", + "820 0.8000000715255737\n", + "830 0.9333333969116211\n", + "avg acc: 0.8121503265641576\n", + ">>test: 0.8781375387589708\n", + "0 0.9333333969116211\n", + "10 0.9000000357627869\n", + "20 0.9333333969116211\n", + "30 0.7666667103767395\n", + "40 0.8666667342185974\n", + "50 0.9000000357627869\n", + "60 0.9666666984558105\n", + "70 0.8333333730697632\n", + "80 0.8000000715255737\n", + "90 0.9000000357627869\n", + "100 0.9666666984558105\n", + "110 0.8333333730697632\n", + "120 0.9666666984558105\n", + "130 0.8666667342185974\n", + "140 0.8666667342185974\n", + "150 0.9333333969116211\n", + "160 0.8000000715255737\n", + "170 0.7666667103767395\n", + "180 0.9333333969116211\n", + "190 0.8000000715255737\n", + "200 0.9000000357627869\n", + "210 0.9333333969116211\n", + "220 0.9000000357627869\n", + "230 0.7666667103767395\n", + "240 0.9666666984558105\n", + "250 0.8666667342185974\n", + "260 0.9666666984558105\n", + "270 0.9666666984558105\n", + "280 0.9000000357627869\n", + "290 0.8666667342185974\n", + "300 0.9000000357627869\n", + "310 0.9333333969116211\n", + "320 0.9000000357627869\n", + "330 0.8666667342185974\n", + "340 0.9333333969116211\n", + "350 0.9000000357627869\n", + "360 0.9000000357627869\n", + "370 0.9333333969116211\n", + "380 0.9333333969116211\n", + "390 0.8333333730697632\n", + "400 0.9333333969116211\n", + "410 0.9333333969116211\n", + "420 0.8666667342185974\n", + "430 0.8333333730697632\n", + "440 0.9000000357627869\n", + "450 0.9666666984558105\n", + "460 0.8333333730697632\n", + "470 0.9000000357627869\n", + "480 0.8333333730697632\n", + "490 0.9000000357627869\n", + "500 0.8666667342185974\n", + "510 0.9333333969116211\n", + "520 0.8666667342185974\n", + "530 0.8333333730697632\n", + "540 0.9000000357627869\n", + "550 0.9666666984558105\n", + "560 0.9666666984558105\n", + "570 0.9000000357627869\n", + "580 0.9333333969116211\n", + "590 0.8333333730697632\n", + "600 0.8333333730697632\n", + "610 0.9666666984558105\n", + "620 0.9000000357627869\n", + "630 0.9333333969116211\n", + "640 0.9666666984558105\n", + "650 0.9333333969116211\n", + "660 0.9666666984558105\n", + "670 0.9333333969116211\n", + "680 0.8000000715255737\n", + "690 0.9000000357627869\n", + "700 0.9000000357627869\n", + "710 0.8666667342185974\n", + "720 0.9333333969116211\n", + "730 0.7666667103767395\n", + "740 0.9333333969116211\n", + "750 0.9333333969116211\n", + "760 0.9666666984558105\n", + "770 0.8000000715255737\n", + "780 0.9333333969116211\n", + "790 0.9000000357627869\n", + "800 0.9333333969116211\n", + "810 0.9333333969116211\n", + "820 0.9333333969116211\n", + "830 0.8000000715255737\n", + "avg acc: 0.89184657182339\n", + ">>test: 0.8595524055780552\n", + "0 0.9000000357627869\n", + "10 0.9000000357627869\n", + "20 0.9333333969116211\n", + "30 0.9333333969116211\n", + "40 0.9333333969116211\n", + "50 0.8666667342185974\n", + "60 0.8333333730697632\n", + "70 0.9000000357627869\n", + "80 0.8666667342185974\n", + "90 0.8666667342185974\n", + "100 0.9666666984558105\n", + "110 0.9666666984558105\n", + "120 1.0\n", + "130 0.8666667342185974\n", + "140 0.9666666984558105\n", + "150 0.9666666984558105\n", + "160 0.9000000357627869\n", + "170 0.9333333969116211\n", + "180 0.9000000357627869\n", + "190 0.9000000357627869\n", + "200 0.8333333730697632\n", + "210 0.9333333969116211\n", + "220 0.9333333969116211\n", + "230 0.9000000357627869\n", + "240 0.8333333730697632\n", + "250 0.8666667342185974\n", + "260 0.9666666984558105\n", + "270 0.9666666984558105\n", + "280 0.9333333969116211\n", + "290 0.9666666984558105\n", + "300 0.9000000357627869\n", + "310 0.9333333969116211\n", + "320 0.9333333969116211\n", + "330 0.9000000357627869\n", + "340 0.9000000357627869\n", + "350 0.9000000357627869\n", + "360 0.9666666984558105\n", + "370 0.8666667342185974\n", + "380 0.9666666984558105\n", + "390 0.8333333730697632\n", + "400 0.9333333969116211\n", + "410 0.9000000357627869\n", + "420 0.8333333730697632\n", + "430 0.9000000357627869\n", + "440 0.9000000357627869\n", + "450 0.9000000357627869\n", + "460 0.9333333969116211\n", + "470 0.8000000715255737\n", + "480 0.9333333969116211\n", + "490 0.9333333969116211\n", + "500 0.9666666984558105\n", + "510 0.9333333969116211\n", + "520 0.9333333969116211\n", + "530 0.8666667342185974\n", + "540 0.9000000357627869\n", + "550 0.9333333969116211\n", + "560 0.9000000357627869\n", + "570 0.9000000357627869\n", + "580 0.9333333969116211\n", + "590 0.8000000715255737\n", + "600 0.8666667342185974\n", + "610 0.9666666984558105\n", + "620 0.9333333969116211\n", + "630 0.9000000357627869\n", + "640 0.9333333969116211\n", + "650 0.9000000357627869\n", + "660 0.9333333969116211\n", + "670 0.8000000715255737\n", + "680 0.9000000357627869\n", + "690 0.9000000357627869\n", + "700 0.9000000357627869\n", + "710 0.8000000715255737\n", + "720 0.8666667342185974\n", + "730 0.9666666984558105\n", + "740 0.9666666984558105\n", + "750 0.9666666984558105\n", + "760 0.9333333969116211\n", + "770 0.9000000357627869\n", + "780 0.8666667342185974\n", + "790 0.8666667342185974\n", + "800 0.9333333969116211\n", + "810 0.9333333969116211\n", + "820 0.9333333969116211\n", + "830 0.8666667342185974\n", + "avg acc: 0.9173062009205349\n", + ">>test: 0.8850919751526355\n", + "0 0.8666667342185974\n", + "10 0.9000000357627869\n", + "20 0.8666667342185974\n", + "30 0.8666667342185974\n", + "40 0.9666666984558105\n", + "50 0.9333333969116211\n", + "60 0.9000000357627869\n", + "70 0.9666666984558105\n", + "80 1.0\n", + "90 0.9333333969116211\n", + "100 0.8666667342185974\n", + "110 0.8333333730697632\n", + "120 0.9000000357627869\n", + "130 0.9666666984558105\n", + "140 1.0\n", + "150 1.0\n", + "160 0.9333333969116211\n", + "170 0.7333333492279053\n", + "180 0.9000000357627869\n", + "190 0.9000000357627869\n", + "200 1.0\n", + "210 0.9666666984558105\n", + "220 0.9000000357627869\n", + "230 0.8666667342185974\n", + "240 0.9333333969116211\n", + "250 0.9333333969116211\n", + "260 0.9000000357627869\n", + "270 0.9666666984558105\n", + "280 0.9333333969116211\n", + "290 0.9333333969116211\n", + "300 0.9333333969116211\n", + "310 0.9333333969116211\n", + "320 0.8333333730697632\n", + "330 0.8666667342185974\n", + "340 0.9000000357627869\n", + "350 0.9666666984558105\n", + "360 0.8000000715255737\n", + "370 0.9666666984558105\n", + "380 0.9666666984558105\n", + "390 0.9333333969116211\n", + "400 1.0\n", + "410 0.9666666984558105\n", + "420 1.0\n", + "430 0.9666666984558105\n", + "440 1.0\n", + "450 0.8333333730697632\n", + "460 0.8666667342185974\n", + "470 0.9666666984558105\n", + "480 0.8666667342185974\n", + "490 0.9000000357627869\n", + "500 0.9666666984558105\n", + "510 0.9333333969116211\n", + "520 0.9666666984558105\n", + "530 0.9000000357627869\n", + "540 0.9666666984558105\n", + "550 0.9333333969116211\n", + "560 0.9666666984558105\n", + "570 0.9333333969116211\n", + "580 0.9666666984558105\n", + "590 1.0\n", + "600 1.0\n", + "610 0.9000000357627869\n", + "620 0.9333333969116211\n", + "630 0.8666667342185974\n", + "640 0.9666666984558105\n", + "650 0.9333333969116211\n", + "660 1.0\n", + "670 0.9000000357627869\n", + "680 0.9666666984558105\n", + "690 0.8666667342185974\n", + "700 0.9333333969116211\n", + "710 0.8666667342185974\n", + "720 1.0\n", + "730 0.9333333969116211\n", + "740 0.9666666984558105\n", + "750 0.9666666984558105\n", + "760 0.9666666984558105\n", + "770 1.0\n", + "780 1.0\n", + "790 0.9000000357627869\n", + "800 0.9333333969116211\n", + "810 0.9333333969116211\n", + "820 0.9666666984558105\n", + "830 0.9333333969116211\n", + "avg acc: 0.930255836863026\n", + ">>test: 0.8738209919320593\n", + "0 0.9333333969116211\n", + "10 1.0\n", + "20 0.9000000357627869\n", + "30 0.9666666984558105\n", + "40 0.8000000715255737\n", + "50 0.9666666984558105\n", + "60 0.9666666984558105\n", + "70 1.0\n", + "80 0.9666666984558105\n", + "90 0.9666666984558105\n", + "100 0.9000000357627869\n", + "110 0.9000000357627869\n", + "120 0.9666666984558105\n", + "130 0.9000000357627869\n", + "140 0.9333333969116211\n", + "150 0.9000000357627869\n", + "160 0.9666666984558105\n", + "170 1.0\n", + "180 0.9666666984558105\n", + "190 1.0\n", + "200 0.9666666984558105\n", + "210 0.9666666984558105\n", + "220 0.9000000357627869\n", + "230 0.9333333969116211\n", + "240 0.9000000357627869\n", + "250 0.9666666984558105\n", + "260 0.9666666984558105\n", + "270 0.9666666984558105\n", + "280 0.9333333969116211\n", + "290 0.9333333969116211\n", + "300 0.9333333969116211\n", + "310 0.9666666984558105\n", + "320 0.9333333969116211\n", + "330 0.9666666984558105\n", + "340 0.9666666984558105\n", + "350 0.8666667342185974\n", + "360 0.9333333969116211\n", + "370 0.9666666984558105\n", + "380 0.8333333730697632\n", + "390 0.9333333969116211\n", + "400 0.8666667342185974\n", + "410 0.9666666984558105\n", + "420 0.9333333969116211\n", + "430 0.9666666984558105\n", + "440 0.9000000357627869\n", + "450 0.9000000357627869\n", + "460 0.9333333969116211\n", + "470 0.9333333969116211\n", + "480 0.9666666984558105\n", + "490 1.0\n", + "500 0.9333333969116211\n", + "510 0.9666666984558105\n", + "520 0.9333333969116211\n", + "530 0.9666666984558105\n", + "540 0.8333333730697632\n", + "550 0.9000000357627869\n", + "560 0.9666666984558105\n", + "570 0.9000000357627869\n", + "580 0.9666666984558105\n", + "590 1.0\n", + "600 0.9000000357627869\n", + "610 0.9333333969116211\n", + "620 1.0\n", + "630 0.9333333969116211\n", + "640 0.9666666984558105\n", + "650 0.9666666984558105\n", + "660 0.9666666984558105\n", + "670 0.9333333969116211\n", + "680 0.8666667342185974\n", + "690 0.9333333969116211\n", + "700 0.9666666984558105\n", + "710 0.9666666984558105\n", + "720 0.9333333969116211\n", + "730 0.9000000357627869\n", + "740 0.9666666984558105\n", + "750 1.0\n", + "760 0.8666667342185974\n", + "770 1.0\n", + "780 0.8666667342185974\n", + "790 0.9333333969116211\n", + "800 0.9333333969116211\n", + "810 0.9666666984558105\n", + "820 1.0\n", + "830 0.9333333969116211\n", + "avg acc: 0.9423661449258562\n", + ">>test: 0.8875300253180863\n", + "0 0.9666666984558105\n", + "10 1.0\n", + "20 0.9666666984558105\n", + "30 0.9666666984558105\n", + "40 0.9333333969116211\n", + "50 0.9333333969116211\n", + "60 0.9333333969116211\n", + "70 0.9000000357627869\n", + "80 0.9333333969116211\n", + "90 0.9333333969116211\n", + "100 1.0\n", + "110 0.9666666984558105\n", + "120 0.8666667342185974\n", + "130 0.9666666984558105\n", + "140 0.9000000357627869\n", + "150 0.9333333969116211\n", + "160 0.9333333969116211\n", + "170 1.0\n", + "180 0.9666666984558105\n", + "190 0.9333333969116211\n", + "200 0.9000000357627869\n", + "210 0.9666666984558105\n", + "220 1.0\n", + "230 0.9666666984558105\n", + "240 0.9333333969116211\n", + "250 0.9666666984558105\n", + "260 0.9000000357627869\n", + "270 0.9666666984558105\n", + "280 0.9333333969116211\n", + "290 0.9333333969116211\n", + "300 0.9666666984558105\n", + "310 1.0\n", + "320 0.9666666984558105\n", + "330 0.9666666984558105\n", + "340 1.0\n", + "350 0.9666666984558105\n", + "360 1.0\n", + "370 0.9666666984558105\n", + "380 1.0\n", + "390 0.9666666984558105\n", + "400 0.9333333969116211\n", + "410 1.0\n", + "420 0.9666666984558105\n", + "430 0.9333333969116211\n", + "440 0.8666667342185974\n", + "450 0.8666667342185974\n", + "460 0.9666666984558105\n", + "470 0.9666666984558105\n", + "480 0.9333333969116211\n", + "490 1.0\n", + "500 0.9666666984558105\n", + "510 0.9666666984558105\n", + "520 1.0\n", + "530 0.9666666984558105\n", + "540 1.0\n", + "550 0.9333333969116211\n", + "560 0.9333333969116211\n", + "570 0.9333333969116211\n", + "580 0.9000000357627869\n", + "590 1.0\n", + "600 0.9333333969116211\n", + "610 1.0\n", + "620 1.0\n", + "630 0.9000000357627869\n", + "640 1.0\n", + "650 0.9333333969116211\n", + "660 0.9333333969116211\n", + "670 0.9333333969116211\n", + "680 0.8666667342185974\n", + "690 0.9000000357627869\n", + "700 0.9333333969116211\n", + "710 0.7000000476837158\n", + "720 0.9666666984558105\n", + "730 0.9333333969116211\n", + "740 1.0\n", + "750 1.0\n", + "760 1.0\n", + "770 1.0\n", + "780 0.9666666984558105\n", + "790 1.0\n", + "800 0.9333333969116211\n", + "810 0.8000000715255737\n", + "820 0.9666666984558105\n", + "830 0.9666666984558105\n", + "avg acc: 0.9476419227014629\n", + ">>test: 0.8856914953933918\n", + "0 0.9000000357627869\n", + "10 0.9333333969116211\n", + "20 0.9333333969116211\n", + "30 0.9333333969116211\n", + "40 0.9666666984558105\n", + "50 0.9666666984558105\n", + "60 1.0\n", + "70 0.9666666984558105\n", + "80 1.0\n", + "90 1.0\n", + "100 0.9333333969116211\n", + "110 1.0\n", + "120 1.0\n", + "130 0.9666666984558105\n", + "140 0.9333333969116211\n", + "150 0.9333333969116211\n", + "160 0.9333333969116211\n", + "170 1.0\n", + "180 1.0\n", + "190 0.9666666984558105\n", + "200 0.9666666984558105\n", + "210 0.9333333969116211\n", + "220 0.9666666984558105\n", + "230 0.9666666984558105\n", + "240 1.0\n", + "250 0.8666667342185974\n", + "260 0.9666666984558105\n", + "270 0.9666666984558105\n", + "280 0.9333333969116211\n", + "290 0.9666666984558105\n", + "300 0.9333333969116211\n", + "310 1.0\n", + "320 0.9666666984558105\n", + "330 0.9333333969116211\n", + "340 0.9333333969116211\n", + "350 0.9333333969116211\n", + "360 0.9333333969116211\n", + "370 0.9000000357627869\n", + "380 0.9666666984558105\n", + "390 0.9333333969116211\n", + "400 1.0\n", + "410 0.9666666984558105\n", + "420 1.0\n", + "430 0.9666666984558105\n", + "440 1.0\n", + "450 0.8333333730697632\n", + "460 0.9333333969116211\n", + "470 0.9666666984558105\n", + "480 0.9666666984558105\n", + "490 0.8666667342185974\n", + "500 1.0\n", + "510 0.9000000357627869\n", + "520 0.9666666984558105\n", + "530 0.9000000357627869\n", + "540 0.9333333969116211\n", + "550 1.0\n", + "560 0.9333333969116211\n", + "570 0.9666666984558105\n", + "580 0.9333333969116211\n", + "590 0.9666666984558105\n", + "600 0.9333333969116211\n", + "610 0.9666666984558105\n", + "620 1.0\n", + "630 1.0\n", + "640 0.9666666984558105\n", + "650 0.9666666984558105\n", + "660 0.9666666984558105\n", + "670 0.9666666984558105\n", + "680 1.0\n", + "690 0.9333333969116211\n", + "700 0.9333333969116211\n", + "710 0.9666666984558105\n", + "720 0.9000000357627869\n", + "730 0.9666666984558105\n", + "740 1.0\n", + "750 1.0\n", + "760 1.0\n", + "770 0.9666666984558105\n", + "780 0.9666666984558105\n", + "790 1.0\n", + "800 0.9666666984558105\n", + "810 0.9333333969116211\n", + "820 0.9666666984558105\n", + "830 0.9000000357627869\n", + "avg acc: 0.955435684401926\n", + ">>test: 0.8800959719313706\n", + "0 0.9666666984558105\n", + "10 0.9000000357627869\n", + "20 1.0\n", + "30 0.9666666984558105\n", + "40 1.0\n", + "50 0.9333333969116211\n", + "60 0.8666667342185974\n", + "70 1.0\n", + "80 0.9666666984558105\n", + "90 0.9666666984558105\n", + "100 1.0\n", + "110 0.9666666984558105\n", + "120 1.0\n", + "130 1.0\n", + "140 0.9333333969116211\n", + "150 0.9666666984558105\n", + "160 0.9333333969116211\n", + "170 0.9666666984558105\n", + "180 0.9666666984558105\n", + "190 1.0\n", + "200 1.0\n", + "210 0.9333333969116211\n", + "220 1.0\n", + "230 1.0\n", + "240 1.0\n", + "250 0.9333333969116211\n", + "260 1.0\n", + "270 0.9666666984558105\n", + "280 1.0\n", + "290 0.9666666984558105\n", + "300 1.0\n", + "310 0.9666666984558105\n", + "320 1.0\n", + "330 1.0\n", + "340 0.9333333969116211\n", + "350 0.9333333969116211\n", + "360 1.0\n", + "370 0.9333333969116211\n", + "380 1.0\n", + "390 0.9666666984558105\n", + "400 0.8666667342185974\n", + "410 1.0\n", + "420 0.9666666984558105\n", + "430 0.9666666984558105\n", + "440 1.0\n", + "450 0.9000000357627869\n", + "460 0.9333333969116211\n", + "470 1.0\n", + "480 0.9333333969116211\n", + "490 1.0\n", + "500 0.9666666984558105\n", + "510 0.9333333969116211\n", + "520 0.9333333969116211\n", + "530 1.0\n", + "540 0.9666666984558105\n", + "550 0.9666666984558105\n", + "560 0.9666666984558105\n", + "570 1.0\n", + "580 1.0\n", + "590 0.9333333969116211\n", + "600 0.9666666984558105\n", + "610 0.9666666984558105\n", + "620 0.9333333969116211\n", + "630 0.9000000357627869\n", + "640 0.9000000357627869\n", + "650 0.9666666984558105\n", + "660 0.9666666984558105\n", + "670 0.9666666984558105\n", + "680 0.9666666984558105\n", + "690 0.9666666984558105\n", + "700 0.9000000357627869\n", + "710 0.9000000357627869\n", + "720 1.0\n", + "730 0.9000000357627869\n", + "740 0.9333333969116211\n", + "750 0.9333333969116211\n", + "760 0.9333333969116211\n", + "770 1.0\n", + "780 0.8666667342185974\n", + "790 1.0\n", + "800 0.9666666984558105\n", + "810 1.0\n", + "820 0.9000000357627869\n", + "830 0.9666666984558105\n", + "avg acc: 0.9621503096547344\n", + ">>test: 0.8820943735200438\n", + "0 1.0\n", + "10 1.0\n", + "20 1.0\n", + "30 1.0\n", + "40 1.0\n", + "50 1.0\n", + "60 1.0\n", + "70 0.9666666984558105\n", + "80 0.9333333969116211\n", + "90 0.9666666984558105\n", + "100 0.9666666984558105\n", + "110 0.9666666984558105\n", + "120 0.9333333969116211\n", + "130 1.0\n", + "140 1.0\n", + "150 0.9666666984558105\n", + "160 0.9666666984558105\n", + "170 1.0\n", + "180 1.0\n", + "190 0.9333333969116211\n", + "200 1.0\n", + "210 1.0\n", + "220 0.9666666984558105\n", + "230 1.0\n", + "240 0.9666666984558105\n", + "250 1.0\n", + "260 0.9333333969116211\n", + "270 0.9666666984558105\n", + "280 1.0\n", + "290 1.0\n", + "300 0.9333333969116211\n", + "310 0.9666666984558105\n", + "320 1.0\n", + "330 1.0\n", + "340 1.0\n", + "350 0.9333333969116211\n", + "360 0.9333333969116211\n", + "370 0.9666666984558105\n", + "380 1.0\n", + "390 1.0\n", + "400 0.9666666984558105\n", + "410 0.9666666984558105\n", + "420 1.0\n", + "430 0.9666666984558105\n", + "440 1.0\n", + "450 0.9333333969116211\n", + "460 0.9333333969116211\n", + "470 0.9666666984558105\n", + "480 1.0\n", + "490 0.9666666984558105\n", + "500 0.9666666984558105\n", + "510 1.0\n", + "520 0.9666666984558105\n", + "530 0.9666666984558105\n", + "540 0.9333333969116211\n", + "550 0.9666666984558105\n", + "560 0.9666666984558105\n", + "570 0.9666666984558105\n", + "580 0.9666666984558105\n", + "590 0.9333333969116211\n", + "600 0.9666666984558105\n", + "610 0.9666666984558105\n", + "620 1.0\n", + "630 1.0\n", + "640 1.0\n", + "650 1.0\n", + "660 0.9666666984558105\n", + "670 0.9666666984558105\n", + "680 0.9666666984558105\n", + "690 0.9333333969116211\n", + "700 0.9333333969116211\n", + "710 0.9666666984558105\n", + "720 0.9333333969116211\n", + "730 0.9666666984558105\n", + "740 0.9333333969116211\n", + "750 0.9666666984558105\n", + "760 1.0\n", + "770 0.9333333969116211\n", + "780 0.9333333969116211\n", + "790 0.9666666984558105\n", + "800 1.0\n", + "810 0.9666666984558105\n", + "820 0.9666666984558105\n", + "830 0.9666666984558105\n", + "avg acc: 0.9673461499545786\n" + ], + "name": "stdout" + } + ] + } + ] +} \ No newline at end of file diff --git "a/lesson53-\346\203\205\346\204\237\345\210\206\347\261\273\345\256\236\346\210\230/lstm.py" "b/lesson53-\346\203\205\346\204\237\345\210\206\347\261\273\345\256\236\346\210\230/lstm.py" new file mode 100644 index 0000000..6a82426 --- /dev/null +++ "b/lesson53-\346\203\205\346\204\237\345\210\206\347\261\273\345\256\236\346\210\230/lstm.py" @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +"""lstm + +Automatically generated by Colaboratory. + +Original file is located at + https://colab.research.google.com/drive/1GX0Rqur8T45MSYhLU9MYWAbycfLH4-Fu +""" + +!pip install torch +!pip install torchtext +!python -m spacy download en + + +# K80 gpu for 12 hours +import torch +from torch import nn, optim +from torchtext import data, datasets + +print('GPU:', torch.cuda.is_available()) + +torch.manual_seed(123) + +TEXT = data.Field(tokenize='spacy') +LABEL = data.LabelField(dtype=torch.float) +train_data, test_data = datasets.IMDB.splits(TEXT, LABEL) + +print('len of train data:', len(train_data)) +print('len of test data:', len(test_data)) + +print(train_data.examples[15].text) +print(train_data.examples[15].label) + +# word2vec, glove +TEXT.build_vocab(train_data, max_size=10000, vectors='glove.6B.100d') +LABEL.build_vocab(train_data) + + +batchsz = 30 +device = torch.device('cuda') +train_iterator, test_iterator = data.BucketIterator.splits( + (train_data, test_data), + batch_size = batchsz, + device=device +) + +class RNN(nn.Module): + + def __init__(self, vocab_size, embedding_dim, hidden_dim): + """ + """ + super(RNN, self).__init__() + + # [0-10001] => [100] + self.embedding = nn.Embedding(vocab_size, embedding_dim) + # [100] => [256] + self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers=2, + bidirectional=True, dropout=0.5) + # [256*2] => [1] + self.fc = nn.Linear(hidden_dim*2, 1) + self.dropout = nn.Dropout(0.5) + + + def forward(self, x): + """ + x: [seq_len, b] vs [b, 3, 28, 28] + """ + # [seq, b, 1] => [seq, b, 100] + embedding = self.dropout(self.embedding(x)) + + # output: [seq, b, hid_dim*2] + # hidden/h: [num_layers*2, b, hid_dim] + # cell/c: [num_layers*2, b, hid_di] + output, (hidden, cell) = self.rnn(embedding) + + # [num_layers*2, b, hid_dim] => 2 of [b, hid_dim] => [b, hid_dim*2] + hidden = torch.cat([hidden[-2], hidden[-1]], dim=1) + + # [b, hid_dim*2] => [b, 1] + hidden = self.dropout(hidden) + out = self.fc(hidden) + + return out + +rnn = RNN(len(TEXT.vocab), 100, 256) + +pretrained_embedding = TEXT.vocab.vectors +print('pretrained_embedding:', pretrained_embedding.shape) +rnn.embedding.weight.data.copy_(pretrained_embedding) +print('embedding layer inited.') + +optimizer = optim.Adam(rnn.parameters(), lr=1e-3) +criteon = nn.BCEWithLogitsLoss().to(device) +rnn.to(device) + +import numpy as np + +def binary_acc(preds, y): + """ + get accuracy + """ + preds = torch.round(torch.sigmoid(preds)) + correct = torch.eq(preds, y).float() + acc = correct.sum() / len(correct) + return acc + +def train(rnn, iterator, optimizer, criteon): + + avg_acc = [] + rnn.train() + + for i, batch in enumerate(iterator): + + # [seq, b] => [b, 1] => [b] + pred = rnn(batch.text).squeeze(1) + # + loss = criteon(pred, batch.label) + acc = binary_acc(pred, batch.label).item() + avg_acc.append(acc) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + if i%10 == 0: + print(i, acc) + + avg_acc = np.array(avg_acc).mean() + print('avg acc:', avg_acc) + + +def eval(rnn, iterator, criteon): + + avg_acc = [] + + rnn.eval() + + with torch.no_grad(): + for batch in iterator: + + # [b, 1] => [b] + pred = rnn(batch.text).squeeze(1) + + # + loss = criteon(pred, batch.label) + + acc = binary_acc(pred, batch.label).item() + avg_acc.append(acc) + + avg_acc = np.array(avg_acc).mean() + + print('>>test:', avg_acc) + +for epoch in range(10): + + eval(rnn, test_iterator, criteon) + train(rnn, train_iterator, optimizer, criteon) \ No newline at end of file diff --git "a/lesson54-61-GAN\345\216\237\347\220\206/GAN\346\240\267\347\211\207.jpg" "b/lesson54-61-GAN\345\216\237\347\220\206/GAN\346\240\267\347\211\207.jpg" new file mode 100644 index 0000000..ece21fc Binary files /dev/null and "b/lesson54-61-GAN\345\216\237\347\220\206/GAN\346\240\267\347\211\207.jpg" differ diff --git "a/lesson54-61-GAN\345\216\237\347\220\206/\346\267\261\345\272\246\345\255\246\344\271\240\357\274\232GAN.pdf" "b/lesson54-61-GAN\345\216\237\347\220\206/\346\267\261\345\272\246\345\255\246\344\271\240\357\274\232GAN.pdf" new file mode 100644 index 0000000..20d9800 Binary files /dev/null and "b/lesson54-61-GAN\345\216\237\347\220\206/\346\267\261\345\272\246\345\255\246\344\271\240\357\274\232GAN.pdf" differ diff --git "a/lesson6-\345\237\272\346\234\254\346\225\260\346\215\256\347\261\273\345\236\213/lesson6.pdf" "b/lesson6-\345\237\272\346\234\254\346\225\260\346\215\256\347\261\273\345\236\213/lesson6.pdf" new file mode 100644 index 0000000..06e8260 Binary files /dev/null and "b/lesson6-\345\237\272\346\234\254\346\225\260\346\215\256\347\261\273\345\236\213/lesson6.pdf" differ diff --git "a/lesson62-65-GAN\345\256\236\346\210\230/gan.py" "b/lesson62-65-GAN\345\256\236\346\210\230/gan.py" new file mode 100644 index 0000000..e17bb4c --- /dev/null +++ "b/lesson62-65-GAN\345\256\236\346\210\230/gan.py" @@ -0,0 +1,248 @@ +import torch +from torch import nn, optim, autograd +import numpy as np +import visdom +from torch.nn import functional as F +from matplotlib import pyplot as plt +import random + +h_dim = 400 +batchsz = 512 +viz = visdom.Visdom() + +class Generator(nn.Module): + + def __init__(self): + super(Generator, self).__init__() + + self.net = nn.Sequential( + nn.Linear(2, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, 2), + ) + + def forward(self, z): + output = self.net(z) + return output + + +class Discriminator(nn.Module): + + def __init__(self): + super(Discriminator, self).__init__() + + self.net = nn.Sequential( + nn.Linear(2, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, 1), + nn.Sigmoid() + ) + + def forward(self, x): + output = self.net(x) + return output.view(-1) + +def data_generator(): + + scale = 2. + centers = [ + (1, 0), + (-1, 0), + (0, 1), + (0, -1), + (1. / np.sqrt(2), 1. / np.sqrt(2)), + (1. / np.sqrt(2), -1. / np.sqrt(2)), + (-1. / np.sqrt(2), 1. / np.sqrt(2)), + (-1. / np.sqrt(2), -1. / np.sqrt(2)) + ] + centers = [(scale * x, scale * y) for x, y in centers] + while True: + dataset = [] + for i in range(batchsz): + point = np.random.randn(2) * .02 + center = random.choice(centers) + point[0] += center[0] + point[1] += center[1] + dataset.append(point) + dataset = np.array(dataset, dtype='float32') + dataset /= 1.414 # stdev + yield dataset + + # for i in range(100000//25): + # for x in range(-2, 3): + # for y in range(-2, 3): + # point = np.random.randn(2).astype(np.float32) * 0.05 + # point[0] += 2 * x + # point[1] += 2 * y + # dataset.append(point) + # + # dataset = np.array(dataset) + # print('dataset:', dataset.shape) + # viz.scatter(dataset, win='dataset', opts=dict(title='dataset', webgl=True)) + # + # while True: + # np.random.shuffle(dataset) + # + # for i in range(len(dataset)//batchsz): + # yield dataset[i*batchsz : (i+1)*batchsz] + + +def generate_image(D, G, xr, epoch): + """ + Generates and saves a plot of the true distribution, the generator, and the + critic. + """ + N_POINTS = 128 + RANGE = 3 + plt.clf() + + points = np.zeros((N_POINTS, N_POINTS, 2), dtype='float32') + points[:, :, 0] = np.linspace(-RANGE, RANGE, N_POINTS)[:, None] + points[:, :, 1] = np.linspace(-RANGE, RANGE, N_POINTS)[None, :] + points = points.reshape((-1, 2)) + # (16384, 2) + # print('p:', points.shape) + + # draw contour + with torch.no_grad(): + points = torch.Tensor(points).cuda() # [16384, 2] + disc_map = D(points).cpu().numpy() # [16384] + x = y = np.linspace(-RANGE, RANGE, N_POINTS) + cs = plt.contour(x, y, disc_map.reshape((len(x), len(y))).transpose()) + plt.clabel(cs, inline=1, fontsize=10) + # plt.colorbar() + + + # draw samples + with torch.no_grad(): + z = torch.randn(batchsz, 2).cuda() # [b, 2] + samples = G(z).cpu().numpy() # [b, 2] + plt.scatter(xr[:, 0], xr[:, 1], c='orange', marker='.') + plt.scatter(samples[:, 0], samples[:, 1], c='green', marker='+') + + viz.matplot(plt, win='contour', opts=dict(title='p(x):%d'%epoch)) + + +def weights_init(m): + if isinstance(m, nn.Linear): + # m.weight.data.normal_(0.0, 0.02) + nn.init.kaiming_normal_(m.weight) + m.bias.data.fill_(0) + +def gradient_penalty(D, xr, xf): + """ + + :param D: + :param xr: + :param xf: + :return: + """ + LAMBDA = 0.3 + + # only constrait for Discriminator + xf = xf.detach() + xr = xr.detach() + + # [b, 1] => [b, 2] + alpha = torch.rand(batchsz, 1).cuda() + alpha = alpha.expand_as(xr) + + interpolates = alpha * xr + ((1 - alpha) * xf) + interpolates.requires_grad_() + + disc_interpolates = D(interpolates) + + gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates, + grad_outputs=torch.ones_like(disc_interpolates), + create_graph=True, retain_graph=True, only_inputs=True)[0] + + gp = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * LAMBDA + + return gp + +def main(): + + torch.manual_seed(23) + np.random.seed(23) + + G = Generator().cuda() + D = Discriminator().cuda() + G.apply(weights_init) + D.apply(weights_init) + + optim_G = optim.Adam(G.parameters(), lr=1e-3, betas=(0.5, 0.9)) + optim_D = optim.Adam(D.parameters(), lr=1e-3, betas=(0.5, 0.9)) + + + data_iter = data_generator() + print('batch:', next(data_iter).shape) + + viz.line([[0,0]], [0], win='loss', opts=dict(title='loss', + legend=['D', 'G'])) + + for epoch in range(50000): + + # 1. train discriminator for k steps + for _ in range(5): + x = next(data_iter) + xr = torch.from_numpy(x).cuda() + + # [b] + predr = (D(xr)) + # max log(lossr) + lossr = - (predr.mean()) + + # [b, 2] + z = torch.randn(batchsz, 2).cuda() + # stop gradient on G + # [b, 2] + xf = G(z).detach() + # [b] + predf = (D(xf)) + # min predf + lossf = (predf.mean()) + + # gradient penalty + gp = gradient_penalty(D, xr, xf) + + loss_D = lossr + lossf + gp + optim_D.zero_grad() + loss_D.backward() + # for p in D.parameters(): + # print(p.grad.norm()) + optim_D.step() + + + # 2. train Generator + z = torch.randn(batchsz, 2).cuda() + xf = G(z) + predf = (D(xf)) + # max predf + loss_G = - (predf.mean()) + optim_G.zero_grad() + loss_G.backward() + optim_G.step() + + + if epoch % 100 == 0: + viz.line([[loss_D.item(), loss_G.item()]], [epoch], win='loss', update='append') + + generate_image(D, G, xr, epoch) + + print(loss_D.item(), loss_G.item()) + + + + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git "a/lesson62-65-GAN\345\256\236\346\210\230/wgan_gp.py" "b/lesson62-65-GAN\345\256\236\346\210\230/wgan_gp.py" new file mode 100644 index 0000000..c88396b --- /dev/null +++ "b/lesson62-65-GAN\345\256\236\346\210\230/wgan_gp.py" @@ -0,0 +1,248 @@ +import torch +from torch import nn, optim, autograd +import numpy as np +import visdom +from torch.nn import functional as F +from matplotlib import pyplot as plt +import random + +h_dim = 400 +batchsz = 512 +viz = visdom.Visdom() + +class Generator(nn.Module): + + def __init__(self): + super(Generator, self).__init__() + + self.net = nn.Sequential( + nn.Linear(2, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, 2), + ) + + def forward(self, z): + output = self.net(z) + return output + + +class Discriminator(nn.Module): + + def __init__(self): + super(Discriminator, self).__init__() + + self.net = nn.Sequential( + nn.Linear(2, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, h_dim), + nn.ReLU(True), + nn.Linear(h_dim, 1), + nn.Sigmoid() + ) + + def forward(self, x): + output = self.net(x) + return output.view(-1) + +def data_generator(): + + scale = 2. + centers = [ + (1, 0), + (-1, 0), + (0, 1), + (0, -1), + (1. / np.sqrt(2), 1. / np.sqrt(2)), + (1. / np.sqrt(2), -1. / np.sqrt(2)), + (-1. / np.sqrt(2), 1. / np.sqrt(2)), + (-1. / np.sqrt(2), -1. / np.sqrt(2)) + ] + centers = [(scale * x, scale * y) for x, y in centers] + while True: + dataset = [] + for i in range(batchsz): + point = np.random.randn(2) * .02 + center = random.choice(centers) + point[0] += center[0] + point[1] += center[1] + dataset.append(point) + dataset = np.array(dataset, dtype='float32') + dataset /= 1.414 # stdev + yield dataset + + # for i in range(100000//25): + # for x in range(-2, 3): + # for y in range(-2, 3): + # point = np.random.randn(2).astype(np.float32) * 0.05 + # point[0] += 2 * x + # point[1] += 2 * y + # dataset.append(point) + # + # dataset = np.array(dataset) + # print('dataset:', dataset.shape) + # viz.scatter(dataset, win='dataset', opts=dict(title='dataset', webgl=True)) + # + # while True: + # np.random.shuffle(dataset) + # + # for i in range(len(dataset)//batchsz): + # yield dataset[i*batchsz : (i+1)*batchsz] + + +def generate_image(D, G, xr, epoch): + """ + Generates and saves a plot of the true distribution, the generator, and the + critic. + """ + N_POINTS = 128 + RANGE = 3 + plt.clf() + + points = np.zeros((N_POINTS, N_POINTS, 2), dtype='float32') + points[:, :, 0] = np.linspace(-RANGE, RANGE, N_POINTS)[:, None] + points[:, :, 1] = np.linspace(-RANGE, RANGE, N_POINTS)[None, :] + points = points.reshape((-1, 2)) + # (16384, 2) + # print('p:', points.shape) + + # draw contour + with torch.no_grad(): + points = torch.Tensor(points).cuda() # [16384, 2] + disc_map = D(points).cpu().numpy() # [16384] + x = y = np.linspace(-RANGE, RANGE, N_POINTS) + cs = plt.contour(x, y, disc_map.reshape((len(x), len(y))).transpose()) + plt.clabel(cs, inline=1, fontsize=10) + # plt.colorbar() + + + # draw samples + with torch.no_grad(): + z = torch.randn(batchsz, 2).cuda() # [b, 2] + samples = G(z).cpu().numpy() # [b, 2] + plt.scatter(xr[:, 0], xr[:, 1], c='orange', marker='.') + plt.scatter(samples[:, 0], samples[:, 1], c='green', marker='+') + + viz.matplot(plt, win='contour', opts=dict(title='p(x):%d'%epoch)) + + +def weights_init(m): + if isinstance(m, nn.Linear): + # m.weight.data.normal_(0.0, 0.02) + nn.init.kaiming_normal_(m.weight) + m.bias.data.fill_(0) + +def gradient_penalty(D, xr, xf): + """ + + :param D: + :param xr: + :param xf: + :return: + """ + LAMBDA = 0.3 + + # only constrait for Discriminator + xf = xf.detach() + xr = xr.detach() + + # [b, 1] => [b, 2] + alpha = torch.rand(batchsz, 1).cuda() + alpha = alpha.expand_as(xr) + + interpolates = alpha * xr + ((1 - alpha) * xf) + interpolates.requires_grad_() + + disc_interpolates = D(interpolates) + + gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates, + grad_outputs=torch.ones_like(disc_interpolates), + create_graph=True, retain_graph=True, only_inputs=True)[0] + + gp = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * LAMBDA + + return gp + +def main(): + + torch.manual_seed(23) + np.random.seed(23) + + G = Generator().cuda() + D = Discriminator().cuda() + G.apply(weights_init) + D.apply(weights_init) + + optim_G = optim.Adam(G.parameters(), lr=1e-3, betas=(0.5, 0.9)) + optim_D = optim.Adam(D.parameters(), lr=1e-3, betas=(0.5, 0.9)) + + + data_iter = data_generator() + print('batch:', next(data_iter).shape) + + viz.line([[0,0]], [0], win='loss', opts=dict(title='loss', + legend=['D', 'G'])) + + for epoch in range(50000): + + # 1. train discriminator for k steps + for _ in range(5): + x = next(data_iter) + xr = torch.from_numpy(x).cuda() + + # [b] + predr = (D(xr)) + # max log(lossr) + lossr = - (predr.mean()) + + # [b, 2] + z = torch.randn(batchsz, 2).cuda() + # stop gradient on G + # [b, 2] + xf = G(z).detach() + # [b] + predf = (D(xf)) + # min predf + lossf = (predf.mean()) + + # gradient penalty + gp = gradient_penalty(D, xr, xf) + + loss_D = lossr + lossf + gp + optim_D.zero_grad() + loss_D.backward() + # for p in D.parameters(): + # print(p.grad.norm()) + optim_D.step() + + + # 2. train Generator + z = torch.randn(batchsz, 2).cuda() + xf = G(z) + predf = (D(xf)) + # max predf + loss_G = - (predf.mean()) + optim_G.zero_grad() + loss_G.backward() + optim_G.step() + + + if epoch % 100 == 0: + viz.line([[loss_D.item(), loss_G.item()]], [epoch], win='loss', update='append') + + generate_image(D, G, xr, epoch) + + print(loss_D.item(), loss_G.item()) + + + + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git "a/lesson7-\345\210\233\345\273\272Tensor/lesson7.pdf" "b/lesson7-\345\210\233\345\273\272Tensor/lesson7.pdf" new file mode 100644 index 0000000..82a08c0 Binary files /dev/null and "b/lesson7-\345\210\233\345\273\272Tensor/lesson7.pdf" differ diff --git "a/lesson7-\345\210\233\345\273\272Tensor/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" "b/lesson7-\345\210\233\345\273\272Tensor/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" new file mode 100644 index 0000000..e69de29 diff --git "a/lesson8-\347\264\242\345\274\225\344\270\216\345\210\207\347\211\207/lesson8.pdf" "b/lesson8-\347\264\242\345\274\225\344\270\216\345\210\207\347\211\207/lesson8.pdf" new file mode 100644 index 0000000..c3d27e3 Binary files /dev/null and "b/lesson8-\347\264\242\345\274\225\344\270\216\345\210\207\347\211\207/lesson8.pdf" differ diff --git "a/lesson8-\347\264\242\345\274\225\344\270\216\345\210\207\347\211\207/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" "b/lesson8-\347\264\242\345\274\225\344\270\216\345\210\207\347\211\207/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" new file mode 100644 index 0000000..e69de29 diff --git "a/lesson9-Tensor\345\217\230\346\215\242/Expand.pdf" "b/lesson9-Tensor\345\217\230\346\215\242/Expand.pdf" new file mode 100644 index 0000000..f9927bb Binary files /dev/null and "b/lesson9-Tensor\345\217\230\346\215\242/Expand.pdf" differ diff --git "a/lesson9-Tensor\345\217\230\346\215\242/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" "b/lesson9-Tensor\345\217\230\346\215\242/\350\277\231\344\270\200\347\253\240\344\273\243\347\240\201\351\207\217\345\260\221\357\274\214\345\220\214\345\255\246\344\273\254\347\205\247\347\235\200\345\206\231.txt" new file mode 100644 index 0000000..e69de29 diff --git "a/\350\257\276\347\250\213\344\273\213\347\273\215/2019-01-12 (6).png" "b/\350\257\276\347\250\213\344\273\213\347\273\215/2019-01-12 (6).png" new file mode 100644 index 0000000..2caa2e9 Binary files /dev/null and "b/\350\257\276\347\250\213\344\273\213\347\273\215/2019-01-12 (6).png" differ diff --git "a/\350\257\276\347\250\213\344\273\213\347\273\215/2019-01-13.png" "b/\350\257\276\347\250\213\344\273\213\347\273\215/2019-01-13.png" new file mode 100644 index 0000000..fe04d10 Binary files /dev/null and "b/\350\257\276\347\250\213\344\273\213\347\273\215/2019-01-13.png" differ