Use poetry & reformat code

example/language_model/pytorch/bigram.py

@@ -4,21 +4,23 @@
 import torch
 from torch.nn.functional import one_hot
 
+
 def generate_mapping(data):
-    data = sorted(list(set(''.join(data))))
+    data = sorted(list(set("".join(data))))
     data = {char: index for index, char in enumerate(data)}
     # marks beginning or end of a word
-    data['.'] = 26
+    data["."] = 26
     return data
 
+
 def main():
-    words = open('./names.txt').read().splitlines()
+    words = open("./names.txt").read().splitlines()
     counter = {}
     stoi = generate_mapping(words)
-    value = Tensor.zeros((28,28), dtype=int32)
+    value = Tensor.zeros((28, 28), dtype=int32)
 
     for w in words:
-        chs = ['.'] + list(w) + ['.']
+        chs = ["."] + list(w) + ["."]
         for ch1, ch2 in zip(chs, chs[1:]):
             # bigram = (ch1, ch2)
             # counter[bigram] = counter.get(bigram, 0) + 1
@@ -27,41 +29,40 @@ def main():
             index2 = stoi[ch2]
             value[index1, index2] += 1
 
-
     value.sum()
     print(value.to(float32))
     print(value[1, :])
 
-    counter = sorted(counter.items(), key = lambda kv: -kv[1])
+    counter = sorted(counter.items(), key=lambda kv: -kv[1])
+
 
 def sample_word(distribution, itos):
     index = 0
-    word = ''
+    word = ""
     while True:
         index = torch.multinomial(distribution[index], num_samples=1, replacement=True).item()
         # print("Sampled index", index, itos[index])
         word += itos[index]
-        if(index == 0):
+        if index == 0:
             break
     return word
 
 
 def pytorch_version():
-    words = open('names.txt', 'r').read().splitlines()
+    words = open("names.txt", "r").read().splitlines()
 
     probabilities = torch.zeros((27, 27), dtype=torch.int32)
-    chars = sorted(list(set(''.join(words))))
-    stoi = {s: i+1 for i,s in enumerate(chars)}
-    stoi['.'] = 0
+    chars = sorted(list(set("".join(words))))
+    stoi = {s: i + 1 for i, s in enumerate(chars)}
+    stoi["."] = 0
     itos = {v: k for k, v in stoi.items()}
 
     for w in words:
-      chs = ['.'] + list(w) + ['.']
-      for ch1, ch2 in zip(chs, chs[1:]):
-        ix1 = stoi[ch1]
-        ix2 = stoi[ch2]
-        probabilities[ix1, ix2] += 1
-
+        chs = ["."] + list(w) + ["."]
+        for ch1, ch2 in zip(chs, chs[1:]):
+            ix1 = stoi[ch1]
+            ix2 = stoi[ch2]
+            probabilities[ix1, ix2] += 1
 
     # Converting N so each row is a probablity distribution
     probabilities = probabilities.float()
@@ -75,23 +76,24 @@ def pytorch_version():
 
     print(words)
 
+
 def Neural_net():
-    words = open('names.txt', 'r').read().splitlines()
-    chars = sorted(list(set(''.join(words))))
-    stoi = {s: i+1 for i,s in enumerate(chars)}
-    stoi['.'] = 0
+    words = open("names.txt", "r").read().splitlines()
+    chars = sorted(list(set("".join(words))))
+    stoi = {s: i + 1 for i, s in enumerate(chars)}
+    stoi["."] = 0
     itos = {v: k for k, v in stoi.items()}
 
     xs, ys = [], []
 
     for w in words:
-      chs = ['.'] + list(w) + ['.']
-      for ch1, ch2 in zip(chs, chs[1:]):
-        ix1 = stoi[ch1]
-        ix2 = stoi[ch2]
+        chs = ["."] + list(w) + ["."]
+        for ch1, ch2 in zip(chs, chs[1:]):
+            ix1 = stoi[ch1]
+            ix2 = stoi[ch2]
 
-        xs.append(ix1)
-        ys.append(ix1)
+            xs.append(ix1)
+            ys.append(ix1)
 
     xs = torch.tensor(xs)
     ys = torch.tensor(ys)

example/language_model/pytorch/language-model.py

@@ -9,38 +9,42 @@
 MINIBATCH_SIZE = 32
 EPOCHS = 100000
 
+
 def generate_mapping(data):
-    chars = sorted(list(set(''.join(data))))
+    chars = sorted(list(set("".join(data))))
     stoi = {char: index + 1 for index, char in enumerate(chars)}
     # marks beginning or end of a word
-    stoi['.'] = 0
+    stoi["."] = 0
     return stoi
 
+
 def generate_learning_rates(size):
     lre = torch.linspace(-6, 0, size)
-    return 10 ** lre # we want the learning rates to be spaced exponentially
+    return 10**lre  # we want the learning rates to be spaced exponentially
+
 
 def load_data():
     data, label = [], []
-    words = open('./names.txt', 'r').read().splitlines()
+    words = open("./names.txt", "r").read().splitlines()
     stoi = generate_mapping(words)
     # itos = {v: k for k, v in stoi.items()}
 
     for w in words:
         context = [0] * CONTEXT_SIZE
-        for ch in w + '.':
-          ix = stoi[ch]
-          data.append(context)
-          label.append(ix)
-          context = context[1:] + [ix] # crop and append
+        for ch in w + ".":
+            ix = stoi[ch]
+            data.append(context)
+            label.append(ix)
+            context = context[1:] + [ix]  # crop and append
 
     data = torch.tensor(data)
     label = torch.tensor(label)
     return data, label
 
+
 def main():
     data, label = load_data()
-    # Creating an embedding from our data with each token being embedding represented 
+    # Creating an embedding from our data with each token being embedding represented
     # by a vector of length "EMBEDDING_SIZE"
     C = torch.rand((27, EMBEDDING_SIZE))
 
@@ -64,7 +68,7 @@ def main():
     avgs = []
 
     for i in range(EPOCHS):
-        # Minibatching 
+        # Minibatching
         minibatch_indexes = torch.randint(0, data.shape[0], (MINIBATCH_SIZE,))
         embedding = C[data[minibatch_indexes]]
 
@@ -79,23 +83,23 @@ def main():
         loss.backward()
 
         # track stats
-        if i % 1000 == 0: # print every once in a while
-          print(f'{i:7d}/{EPOCHS:7d}: {loss.item():.4f}')
-          if i > EPOCHS / 2:
-              avgs.append(loss.item())
+        if i % 1000 == 0:  # print every once in a while
+            print(f"{i:7d}/{EPOCHS:7d}: {loss.item():.4f}")
+            if i > EPOCHS / 2:
+                avgs.append(loss.item())
 
         used_lrs.append(i)
         losses.append(loss.item())
 
-        lr = 0.1 if i < EPOCHS / 2 else 0.01 # step learning rate decay
+        lr = 0.1 if i < EPOCHS / 2 else 0.01  # step learning rate decay
         for p in parameters:
             p.data -= lr * p.grad
 
-
     print("Average loss", sum(avgs) / len(avgs))
     plt.plot(used_lrs, losses)
     plt.legend()
     plt.show()
 
+
 if __name__ == "__main__":
     main()

example/language_model/pytorch/rnn.py

@@ -3,58 +3,62 @@
 import matplotlib.pyplot as plt
 from math import sqrt
 
+
 def generate_mapping(data):
-    chars = sorted(list(set(''.join(data))))
+    chars = sorted(list(set("".join(data))))
     stoi = {char: index + 1 for index, char in enumerate(chars)}
     # marks beginning or end of a word
-    stoi['.'] = 0
+    stoi["."] = 0
     return stoi
 
+
 def generate_learning_rates(size):
     lre = torch.linspace(-6, 0, size)
-    return 10 ** lre # we want the learning rates to be spaced exponentially
+    return 10**lre  # we want the learning rates to be spaced exponentially
+
 
 def load_data(context_size):
     data, label = [], []
-    words = open('./names.txt', 'r').read().splitlines()
+    words = open("./names.txt", "r").read().splitlines()
     stoi = generate_mapping(words)
     # itos = {v: k for k, v in stoi.items()}
 
     for w in words:
         context = [0] * context_size
-        for ch in w + '.':
-          ix = stoi[ch]
-          data.append(context)
-          label.append(ix)
-          context = context[1:] + [ix] # crop and append
+        for ch in w + ".":
+            ix = stoi[ch]
+            data.append(context)
+            label.append(ix)
+            context = context[1:] + [ix]  # crop and append
 
     data = torch.tensor(data)
     label = torch.tensor(label)
     return data, label
 
+
 def main():
     # How much tokens to keep as context when making the prediction for the next one
     CONTEXT_SIZE = 3
     # Size of the vector to represent a single token
     EMBEDDING_SIZE = 10
-    VOCAB_SIZE = 27 # There are 27 possible chars in our dataset
+    VOCAB_SIZE = 27  # There are 27 possible chars in our dataset
 
     data, label = load_data(CONTEXT_SIZE)
-    # Creating an embedding from our data with each token being embedding represented 
+    # Creating an embedding from our data with each token being embedding represented
     # by a vector of length "EMBEDDING_SIZE"
     C = torch.rand((VOCAB_SIZE, EMBEDDING_SIZE))
 
     NUMBER_OF_NEURONS = 200
 
     # Creating hidden layer
     # Using Kaiming init https://pytorch.org/docs/stable/nn.init.html
-    w1 = torch.rand((CONTEXT_SIZE * EMBEDDING_SIZE, NUMBER_OF_NEURONS)) * ((5/3) / (CONTEXT_SIZE*EMBEDDING_SIZE))
-    print("First ",  ((5/3) / (CONTEXT_SIZE*EMBEDDING_SIZE)))
+    w1 = torch.rand((CONTEXT_SIZE * EMBEDDING_SIZE, NUMBER_OF_NEURONS)) * ((5 / 3) / (CONTEXT_SIZE * EMBEDDING_SIZE))
+    print("First ", ((5 / 3) / (CONTEXT_SIZE * EMBEDDING_SIZE)))
     b1 = torch.rand(NUMBER_OF_NEURONS) * 0.01
 
     # Creating the output layer
-    w2 = torch.rand((NUMBER_OF_NEURONS, 27)) *  ((5/3) / (NUMBER_OF_NEURONS))
-    print("second ", ((5/3) * sqrt(NUMBER_OF_NEURONS)))
+    w2 = torch.rand((NUMBER_OF_NEURONS, 27)) * ((5 / 3) / (NUMBER_OF_NEURONS))
+    print("second ", ((5 / 3) * sqrt(NUMBER_OF_NEURONS)))
     b2 = torch.rand(27) * 0.01
 
     parameters = [C, w1, b1, w2, b2]
@@ -70,7 +74,7 @@ def main():
     MINIBATCH_SIZE = 32
     avgs = []
     for i in range(EPOCHS):
-        # Minibatching 
+        # Minibatching
         minibatch_indexes = torch.randint(0, data.shape[0], (MINIBATCH_SIZE,))
         embedding = C[data[minibatch_indexes]]
 
@@ -84,10 +88,10 @@ def main():
         loss.backward()
 
         # track stats
-        if i % 1000 == 0: # print every once in a while
-          print(f'{i:7d}/{EPOCHS:7d}: {loss.item():.4f}')
-          if i > EPOCHS / 2:
-              avgs.append(loss.item())
+        if i % 1000 == 0:  # print every once in a while
+            print(f"{i:7d}/{EPOCHS:7d}: {loss.item():.4f}")
+            if i > EPOCHS / 2:
+                avgs.append(loss.item())
 
         used_lrs.append(i)
         losses.append(loss.item())
@@ -101,5 +105,6 @@ def main():
     plt.legend()
     plt.show()
 
+
 if __name__ == "__main__":
     main()

gigatorch/cnn.py

@@ -29,17 +29,22 @@ def compute(self, input: Tensor) -> Tensor:
 Height: The height of the input.
 Width: The width of the input.
 """
+
+
 class MaxPool2D(Compute):
     def __init__(self, kernel_size, stride=None):
         self.kernel_size = kernel_size
         self.stride = stride if stride is not None else kernel_size
 
-
     def compute(self, input: Tensor) -> Tensor:
         assert len(input.shape) == 4, f"can't 2d pool {input.shape}"
         (batch_size, channels, height, width) = input.shape
-        assert (height - self.kernel_size) % self.stride == 0, f"Height does not fit the kernel size {self.kernel_size} and stride {self.stride}"
-        assert (width - self.kernel_size) % self.stride == 0, f"Width does not fit the kernel size {self.kernel_size} and stride {self.stride}"
+        assert (
+            height - self.kernel_size
+        ) % self.stride == 0, f"Height does not fit the kernel size {self.kernel_size} and stride {self.stride}"
+        assert (
+            width - self.kernel_size
+        ) % self.stride == 0, f"Width does not fit the kernel size {self.kernel_size} and stride {self.stride}"
 
         print("Computing maxpool")
         print("Input shape: ", input.shape)
@@ -85,9 +90,7 @@ def __init__(self, in_channels, out_channels, kernel_size, activation_fn, stride
         self.kernels = Tensor(
             [
                 [
-                    WightInitializer().xavier_uniform(
-                        in_channels, out_channels, kernel_size, kernel_size
-                    )
+                    WightInitializer().xavier_uniform(in_channels, out_channels, kernel_size, kernel_size)
                     for _ in range(in_channels)
                 ]
                 for _ in range(out_channels)
@@ -119,6 +122,7 @@ def compute(self, input):
 
         return output
 
+
 class CNN:
     def __init__(self, train_data_dir, test_data_dir, categories):
         print("Initalising CNN class")

gigatorch/embedding.py

@@ -7,8 +7,7 @@ class Embedding:
     def __init__(self, vocab_size: int, embed_size: int):
         self.vocab_size, self.embed_size = vocab_size, embed_size
         # What should be fan_in fan_out here?
-        self.weight = WightInitializer.xavier_normal(
-            1, 2, vocab_size, embed_size)
+        self.weight = WightInitializer.xavier_normal(1, 2, vocab_size, embed_size)
 
     def __call__(self, idx: Tensor) -> Tensor:
         return (self.vocab_counter == idx.unsqueeze(2)).expand(*idx.shape, self.vocab_size) @ self.weight
@@ -25,7 +24,7 @@ def prepare_data(raw_text, context_size=2):
     data = []
     for i in range(context_size, len(raw_text) - context_size):
         target = raw_text[i]
-        context = raw_text[i - context_size: i + context_size + 1]
+        context = raw_text[i - context_size : i + context_size + 1]
         data.append((target, context))
 
     return data, word_to_index, index_to_word