microsoft/AI-For-Beginners
Publicmirrored from https://github.com/microsoft/AI-For-BeginnersAvailable
lessons/5-NLP/17-GenerativeNetworks/torchnlp.py
115lines · modecode
| 1 | import builtins |
| 2 | import torch |
| 3 | import torchtext |
| 4 | import collections |
| 5 | import os |
| 6 | |
| 7 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") |
| 8 | |
| 9 | vocab = None |
| 10 | tokenizer = torchtext.data.utils.get_tokenizer('basic_english') |
| 11 | |
| 12 | def load_dataset(ngrams=1,min_freq=1): |
| 13 | global vocab, tokenizer |
| 14 | print("Loading dataset...") |
| 15 | train_dataset, test_dataset = torchtext.datasets.AG_NEWS(root='./data') |
| 16 | train_dataset = list(train_dataset) |
| 17 | test_dataset = list(test_dataset) |
| 18 | classes = ['World', 'Sports', 'Business', 'Sci/Tech'] |
| 19 | print('Building vocab...') |
| 20 | counter = collections.Counter() |
| 21 | for (label, line) in train_dataset: |
| 22 | counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams)) |
| 23 | vocab = torchtext.vocab.vocab(counter, min_freq=min_freq) |
| 24 | return train_dataset,test_dataset,classes,vocab |
| 25 | |
| 26 | stoi_hash = {} |
| 27 | def encode(x,voc=None,unk=0,tokenizer=tokenizer): |
| 28 | global stoi_hash |
| 29 | v = vocab if voc is None else voc |
| 30 | if v in stoi_hash.keys(): |
| 31 | stoi = stoi_hash[v] |
| 32 | else: |
| 33 | # Handle both vocab.vocab objects (with get_stoi() method) and GloVe objects (with stoi attribute) |
| 34 | if hasattr(v, 'get_stoi'): |
| 35 | stoi = v.get_stoi() |
| 36 | else: |
| 37 | stoi = v.stoi |
| 38 | stoi_hash[v]=stoi |
| 39 | return [stoi.get(s,unk) for s in tokenizer(x)] |
| 40 | |
| 41 | def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200): |
| 42 | optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr) |
| 43 | loss_fn = loss_fn.to(device) |
| 44 | net.train() |
| 45 | total_loss,acc,count,i = 0,0,0,0 |
| 46 | for labels,features in dataloader: |
| 47 | optimizer.zero_grad() |
| 48 | features, labels = features.to(device), labels.to(device) |
| 49 | out = net(features) |
| 50 | loss = loss_fn(out,labels) #cross_entropy(out,labels) |
| 51 | loss.backward() |
| 52 | optimizer.step() |
| 53 | total_loss+=loss |
| 54 | _,predicted = torch.max(out,1) |
| 55 | acc+=(predicted==labels).sum() |
| 56 | count+=len(labels) |
| 57 | i+=1 |
| 58 | if i%report_freq==0: |
| 59 | print(f"{count}: acc={acc.item()/count}") |
| 60 | if epoch_size and count>epoch_size: |
| 61 | break |
| 62 | return total_loss.item()/count, acc.item()/count |
| 63 | |
| 64 | def padify(b,voc=None,tokenizer=tokenizer): |
| 65 | # b is the list of tuples of length batch_size |
| 66 | # - first element of a tuple = label, |
| 67 | # - second = feature (text sequence) |
| 68 | # build vectorized sequence |
| 69 | v = [encode(x[1],voc=voc,tokenizer=tokenizer) for x in b] |
| 70 | # compute max length of a sequence in this minibatch |
| 71 | l = max(map(len,v)) |
| 72 | return ( # tuple of two tensors - labels and features |
| 73 | torch.LongTensor([t[0]-1 for t in b]), |
| 74 | torch.stack([torch.nn.functional.pad(torch.tensor(t),(0,l-len(t)),mode='constant',value=0) for t in v]) |
| 75 | ) |
| 76 | |
| 77 | def offsetify(b,voc=None): |
| 78 | # first, compute data tensor from all sequences |
| 79 | x = [torch.tensor(encode(t[1],voc=voc)) for t in b] |
| 80 | # now, compute the offsets by accumulating the tensor of sequence lengths |
| 81 | o = [0] + [len(t) for t in x] |
| 82 | o = torch.tensor(o[:-1]).cumsum(dim=0) |
| 83 | return ( |
| 84 | torch.LongTensor([t[0]-1 for t in b]), # labels |
| 85 | torch.cat(x), # text |
| 86 | o |
| 87 | ) |
| 88 | |
| 89 | def train_epoch_emb(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200,use_pack_sequence=False): |
| 90 | optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr) |
| 91 | loss_fn = loss_fn.to(device) |
| 92 | net.train() |
| 93 | total_loss,acc,count,i = 0,0,0,0 |
| 94 | for labels,text,off in dataloader: |
| 95 | optimizer.zero_grad() |
| 96 | labels,text = labels.to(device), text.to(device) |
| 97 | if use_pack_sequence: |
| 98 | off = off.to('cpu') |
| 99 | else: |
| 100 | off = off.to(device) |
| 101 | out = net(text, off) |
| 102 | loss = loss_fn(out,labels) #cross_entropy(out,labels) |
| 103 | loss.backward() |
| 104 | optimizer.step() |
| 105 | total_loss+=loss |
| 106 | _,predicted = torch.max(out,1) |
| 107 | acc+=(predicted==labels).sum() |
| 108 | count+=len(labels) |
| 109 | i+=1 |
| 110 | if i%report_freq==0: |
| 111 | print(f"{count}: acc={acc.item()/count}") |
| 112 | if epoch_size and count>epoch_size: |
| 113 | break |
| 114 | return total_loss.item()/count, acc.item()/count |
| 115 | |
| 116 | |