From 5df8aec66cdd87b893a0ee17b97ca8684262f7f7 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 14 Nov 2019 19:59:43 +0100 Subject: [PATCH] GitHub Action formats our code with psf/black (#1569) * GitHub Action formats our code with psf/black @poyea Your review please. * fixup! Format Python code with psf/black push --- .github/workflows/autoblack.yml | 36 +- ciphers/deterministic_miller_rabin.py | 30 +- ciphers/diffie.py | 15 +- .../binary_tree/basic_binary_tree.py | 4 +- data_structures/binary_tree/treap.py | 1 - data_structures/heap/min_heap.py | 7 +- .../linked_list/doubly_linked_list.py | 4 +- data_structures/linked_list/from_sequence.py | 5 +- data_structures/linked_list/print_reverse.py | 6 +- .../longest_increasing_subsequence.py | 3 +- ...longest_increasing_subsequence_o(nlogn).py | 2 + dynamic_programming/max_sub_array.py | 1 + file_transfer/send_file.py | 4 +- maths/ceil.py | 6 +- maths/factorial_python.py | 2 +- maths/factorial_recursive.py | 2 +- maths/floor.py | 6 +- maths/gaussian.py | 2 +- maths/perfect_square.py | 2 +- neural_network/gan.py | 317 ++++++++----- neural_network/input_data.py | 444 +++++++++--------- other/least_recently_used.py | 10 +- project_euler/problem_20/sol4.py | 2 +- project_euler/problem_99/sol1.py | 6 +- web_programming/get_imdbtop.py | 6 +- 25 files changed, 523 insertions(+), 400 deletions(-) diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml index 98310ac80..dc76d4cee 100644 --- a/.github/workflows/autoblack.yml +++ b/.github/workflows/autoblack.yml @@ -1,34 +1,24 @@ -# GitHub Action that uses Black to reformat the Python code in an incoming pull request. -# If all Python code in the pull request is complient with Black then this Action does nothing. -# Othewrwise, Black is run and its changes are committed back to the incoming pull request. +# GitHub Action that uses Black to reformat Python code (if needed) when doing a git push. +# If all Python code in the repo is complient with Black then this Action does nothing. +# Otherwise, Black is run and its changes are committed to the repo. # https://github.com/cclauss/autoblack -name: autoblack -on: [pull_request] +name: autoblack_push +on: [push] jobs: build: runs-on: ubuntu-latest - strategy: - max-parallel: 1 - matrix: - python-version: [3.7] steps: - uses: actions/checkout@v1 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install psf/black - run: pip install black - - name: Run black --check . - run: black --check . - - name: If needed, commit black changes to the pull request + - uses: actions/setup-python@v1 + - run: pip install black + - run: black --check . + - name: If needed, commit black changes to a new pull request if: failure() run: | black . - git config --global user.name 'autoblack' - git config --global user.email 'cclauss@users.noreply.github.com' + git config --global user.name github-actions + git config --global user.email '${GITHUB_ACTOR}@users.noreply.github.com' git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY - git checkout $GITHUB_HEAD_REF - git commit -am "fixup: Format Python code with psf/black" - git push + git commit -am "fixup! Format Python code with psf/black push" + git push --force origin HEAD:$GITHUB_REF diff --git a/ciphers/deterministic_miller_rabin.py b/ciphers/deterministic_miller_rabin.py index 37845d6c9..e604a7b84 100644 --- a/ciphers/deterministic_miller_rabin.py +++ b/ciphers/deterministic_miller_rabin.py @@ -41,19 +41,21 @@ def miller_rabin(n, allow_probable=False): "A return value of True indicates a probable prime." ) # array bounds provided by analysis - bounds = [2_047, - 1_373_653, - 25_326_001, - 3_215_031_751, - 2_152_302_898_747, - 3_474_749_660_383, - 341_550_071_728_321, - 1, - 3_825_123_056_546_413_051, - 1, - 1, - 318_665_857_834_031_151_167_461, - 3_317_044_064_679_887_385_961_981] + bounds = [ + 2_047, + 1_373_653, + 25_326_001, + 3_215_031_751, + 2_152_302_898_747, + 3_474_749_660_383, + 341_550_071_728_321, + 1, + 3_825_123_056_546_413_051, + 1, + 1, + 318_665_857_834_031_151_167_461, + 3_317_044_064_679_887_385_961_981, + ] primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41] for idx, _p in enumerate(bounds, 1): @@ -131,5 +133,5 @@ def test_miller_rabin(): # upper limit for probabilistic test -if __name__ == '__main__': +if __name__ == "__main__": test_miller_rabin() diff --git a/ciphers/diffie.py b/ciphers/diffie.py index 6b0cca1f4..c349aaa2f 100644 --- a/ciphers/diffie.py +++ b/ciphers/diffie.py @@ -1,8 +1,8 @@ def find_primitive(n): for r in range(1, n): li = [] - for x in range(n-1): - val = pow(r,x,n) + for x in range(n - 1): + val = pow(r, x, n) if val in li: break li.append(val) @@ -11,16 +11,15 @@ def find_primitive(n): if __name__ == "__main__": - q = int(input('Enter a prime number q: ')) + q = int(input("Enter a prime number q: ")) a = find_primitive(q) - a_private = int(input('Enter private key of A: ')) + a_private = int(input("Enter private key of A: ")) a_public = pow(a, a_private, q) - b_private = int(input('Enter private key of B: ')) + b_private = int(input("Enter private key of B: ")) b_public = pow(a, b_private, q) a_secret = pow(b_public, a_private, q) b_secret = pow(a_public, b_private, q) - print('The key value generated by A is: ', a_secret) - print('The key value generated by B is: ', b_secret) - + print("The key value generated by A is: ", a_secret) + print("The key value generated by B is: ", b_secret) diff --git a/data_structures/binary_tree/basic_binary_tree.py b/data_structures/binary_tree/basic_binary_tree.py index 6b7de7803..4257a8e3c 100644 --- a/data_structures/binary_tree/basic_binary_tree.py +++ b/data_structures/binary_tree/basic_binary_tree.py @@ -22,7 +22,7 @@ def display(tree): # In Order traversal of the tree def depth_of_tree( - tree + tree, ): # This is the recursive function to find the depth of binary tree. if tree is None: return 0 @@ -36,7 +36,7 @@ def depth_of_tree( def is_full_binary_tree( - tree + tree, ): # This functions returns that is it full binary tree or not? if tree is None: return True diff --git a/data_structures/binary_tree/treap.py b/data_structures/binary_tree/treap.py index b603eec3e..6bc2403f7 100644 --- a/data_structures/binary_tree/treap.py +++ b/data_structures/binary_tree/treap.py @@ -172,7 +172,6 @@ def main(): args = input() print("good by!") - if __name__ == "__main__": diff --git a/data_structures/heap/min_heap.py b/data_structures/heap/min_heap.py index 6184d83be..e68853837 100644 --- a/data_structures/heap/min_heap.py +++ b/data_structures/heap/min_heap.py @@ -77,9 +77,10 @@ class MinHeap: if smallest != idx: array[idx], array[smallest] = array[smallest], array[idx] - self.idx_of_element[array[idx]], self.idx_of_element[ - array[smallest] - ] = ( + ( + self.idx_of_element[array[idx]], + self.idx_of_element[array[smallest]], + ) = ( self.idx_of_element[array[smallest]], self.idx_of_element[array[idx]], ) diff --git a/data_structures/linked_list/doubly_linked_list.py b/data_structures/linked_list/doubly_linked_list.py index 38fff867b..2a95a0045 100644 --- a/data_structures/linked_list/doubly_linked_list.py +++ b/data_structures/linked_list/doubly_linked_list.py @@ -23,9 +23,7 @@ class LinkedList: # making main class named linked list def deleteHead(self): temp = self.head self.head = self.head.next # oldHead <--> 2ndElement(head) - self.head.previous = ( - None - ) # oldHead --> 2ndElement(head) nothing pointing at it so the old head will be removed + self.head.previous = None # oldHead --> 2ndElement(head) nothing pointing at it so the old head will be removed if self.head is None: self.tail = None # if empty linked list return temp diff --git a/data_structures/linked_list/from_sequence.py b/data_structures/linked_list/from_sequence.py index e6d335e81..94b44f150 100644 --- a/data_structures/linked_list/from_sequence.py +++ b/data_structures/linked_list/from_sequence.py @@ -1,6 +1,7 @@ # Recursive Prorgam to create a Linked List from a sequence and # print a string representation of it. + class Node: def __init__(self, data=None): self.data = data @@ -17,7 +18,6 @@ class Node: return string_rep - def make_linked_list(elements_list): """Creates a Linked List from the elements of the given sequence (list/tuple) and returns the head of the Linked List.""" @@ -36,8 +36,7 @@ def make_linked_list(elements_list): return head - -list_data = [1,3,5,32,44,12,43] +list_data = [1, 3, 5, 32, 44, 12, 43] print(f"List: {list_data}") print("Creating Linked List from List.") linked_list = make_linked_list(list_data) diff --git a/data_structures/linked_list/print_reverse.py b/data_structures/linked_list/print_reverse.py index 6572ccd8f..c3a72b6b7 100644 --- a/data_structures/linked_list/print_reverse.py +++ b/data_structures/linked_list/print_reverse.py @@ -1,5 +1,6 @@ # Program to print the elements of a linked list in reverse + class Node: def __init__(self, data=None): self.data = data @@ -16,7 +17,6 @@ class Node: return string_rep - def make_linked_list(elements_list): """Creates a Linked List from the elements of the given sequence (list/tuple) and returns the head of the Linked List.""" @@ -34,6 +34,7 @@ def make_linked_list(elements_list): current = current.next return head + def print_reverse(head_node): """Prints the elements of the given Linked List in reverse order""" @@ -46,8 +47,7 @@ def print_reverse(head_node): print(head_node.data) - -list_data = [14,52,14,12,43] +list_data = [14, 52, 14, 12, 43] linked_list = make_linked_list(list_data) print("Linked List:") print(linked_list) diff --git a/dynamic_programming/longest_increasing_subsequence.py b/dynamic_programming/longest_increasing_subsequence.py index 6d12f1c7c..81b7f8f8f 100644 --- a/dynamic_programming/longest_increasing_subsequence.py +++ b/dynamic_programming/longest_increasing_subsequence.py @@ -48,8 +48,9 @@ def longest_subsequence(array: List[int]) -> List[int]: # This function is recu return temp_array else: return longest_subseq - + if __name__ == "__main__": import doctest + doctest.testmod() diff --git a/dynamic_programming/longest_increasing_subsequence_o(nlogn).py b/dynamic_programming/longest_increasing_subsequence_o(nlogn).py index 4b06e0d88..46790a5a8 100644 --- a/dynamic_programming/longest_increasing_subsequence_o(nlogn).py +++ b/dynamic_programming/longest_increasing_subsequence_o(nlogn).py @@ -6,6 +6,7 @@ ############################# from typing import List + def CeilIndex(v, l, r, key): while r - l > 1: m = (l + r) // 2 @@ -49,4 +50,5 @@ def LongestIncreasingSubsequenceLength(v: List[int]) -> int: if __name__ == "__main__": import doctest + doctest.testmod() diff --git a/dynamic_programming/max_sub_array.py b/dynamic_programming/max_sub_array.py index f7c820971..7350eaf37 100644 --- a/dynamic_programming/max_sub_array.py +++ b/dynamic_programming/max_sub_array.py @@ -75,6 +75,7 @@ if __name__ == "__main__": import time import matplotlib.pyplot as plt from random import randint + inputs = [10, 100, 1000, 10000, 50000, 100000, 200000, 300000, 400000, 500000] tim = [] for i in inputs: diff --git a/file_transfer/send_file.py b/file_transfer/send_file.py index ebc075a30..6494114a9 100644 --- a/file_transfer/send_file.py +++ b/file_transfer/send_file.py @@ -2,8 +2,8 @@ if __name__ == "__main__": import socket # Import socket module ONE_CONNECTION_ONLY = ( - True - ) # Set this to False if you wish to continuously accept connections + True # Set this to False if you wish to continuously accept connections + ) filename = "mytext.txt" port = 12312 # Reserve a port for your service. diff --git a/maths/ceil.py b/maths/ceil.py index 3e46f1474..ff136f685 100644 --- a/maths/ceil.py +++ b/maths/ceil.py @@ -9,10 +9,12 @@ def ceil(x) -> int: >>> all(ceil(n) == math.ceil(n) for n in (1, -1, 0, -0, 1.1, -1.1, 1.0, -1.0, 1_000_000_000)) True """ - return x if isinstance(x, int) or x - int(x) == 0 else int(x + 1) if x > 0 else int(x) + return ( + x if isinstance(x, int) or x - int(x) == 0 else int(x + 1) if x > 0 else int(x) + ) -if __name__ == '__main__': +if __name__ == "__main__": import doctest doctest.testmod() diff --git a/maths/factorial_python.py b/maths/factorial_python.py index b9adfdbae..46688261a 100644 --- a/maths/factorial_python.py +++ b/maths/factorial_python.py @@ -28,7 +28,7 @@ def factorial(input_number: int) -> int: return result -if __name__ == '__main__': +if __name__ == "__main__": import doctest doctest.testmod() diff --git a/maths/factorial_recursive.py b/maths/factorial_recursive.py index 013560b28..4f7074d16 100644 --- a/maths/factorial_recursive.py +++ b/maths/factorial_recursive.py @@ -24,7 +24,7 @@ def factorial(n: int) -> int: return 1 if n == 0 or n == 1 else n * factorial(n - 1) -if __name__ == '__main__': +if __name__ == "__main__": import doctest doctest.testmod() diff --git a/maths/floor.py b/maths/floor.py index a9b680b37..ae6e5129a 100644 --- a/maths/floor.py +++ b/maths/floor.py @@ -9,10 +9,12 @@ def floor(x) -> int: >>> all(floor(n) == math.floor(n) for n in (1, -1, 0, -0, 1.1, -1.1, 1.0, -1.0, 1_000_000_000)) True """ - return x if isinstance(x, int) or x - int(x) == 0 else int(x) if x > 0 else int(x - 1) + return ( + x if isinstance(x, int) or x - int(x) == 0 else int(x) if x > 0 else int(x - 1) + ) -if __name__ == '__main__': +if __name__ == "__main__": import doctest doctest.testmod() diff --git a/maths/gaussian.py b/maths/gaussian.py index e5f55dfaf..ffea20fb2 100644 --- a/maths/gaussian.py +++ b/maths/gaussian.py @@ -50,7 +50,7 @@ def gaussian(x, mu: float = 0.0, sigma: float = 1.0) -> int: >>> gaussian(2523, mu=234234, sigma=3425) 0.0 """ - return 1 / sqrt(2 * pi * sigma ** 2) * exp(-(x - mu) ** 2 / 2 * sigma ** 2) + return 1 / sqrt(2 * pi * sigma ** 2) * exp(-((x - mu) ** 2) / 2 * sigma ** 2) if __name__ == "__main__": diff --git a/maths/perfect_square.py b/maths/perfect_square.py index 9b868c5de..3e7a1c07a 100644 --- a/maths/perfect_square.py +++ b/maths/perfect_square.py @@ -21,7 +21,7 @@ def perfect_square(num: int) -> bool: return math.sqrt(num) * math.sqrt(num) == num -if __name__ == '__main__': +if __name__ == "__main__": import doctest doctest.testmod() diff --git a/neural_network/gan.py b/neural_network/gan.py index edfff4205..76f46314c 100644 --- a/neural_network/gan.py +++ b/neural_network/gan.py @@ -7,28 +7,42 @@ import input_data random_numer = 42 np.random.seed(random_numer) + + def ReLu(x): - mask = (x>0) * 1.0 - return mask *x + mask = (x > 0) * 1.0 + return mask * x + + def d_ReLu(x): - mask = (x>0) * 1.0 + mask = (x > 0) * 1.0 return mask + def arctan(x): return np.arctan(x) + + def d_arctan(x): return 1 / (1 + x ** 2) + def log(x): - return 1 / ( 1+ np.exp(-1*x)) + return 1 / (1 + np.exp(-1 * x)) + + def d_log(x): return log(x) * (1 - log(x)) + def tanh(x): return np.tanh(x) + + def d_tanh(x): return 1 - np.tanh(x) ** 2 + def plot(samples): fig = plt.figure(figsize=(4, 4)) gs = gridspec.GridSpec(4, 4) @@ -36,104 +50,140 @@ def plot(samples): for i, sample in enumerate(samples): ax = plt.subplot(gs[i]) - plt.axis('off') + plt.axis("off") ax.set_xticklabels([]) ax.set_yticklabels([]) - ax.set_aspect('equal') - plt.imshow(sample.reshape(28, 28), cmap='Greys_r') + ax.set_aspect("equal") + plt.imshow(sample.reshape(28, 28), cmap="Greys_r") return fig - # 1. Load Data and declare hyper -print('--------- Load Data ----------') -mnist = input_data.read_data_sets('MNIST_data', one_hot=False) +print("--------- Load Data ----------") +mnist = input_data.read_data_sets("MNIST_data", one_hot=False) temp = mnist.test images, labels = temp.images, temp.labels -images, labels = shuffle(np.asarray(images),np.asarray(labels)) +images, labels = shuffle(np.asarray(images), np.asarray(labels)) num_epoch = 10 learing_rate = 0.00009 G_input = 100 -hidden_input,hidden_input2,hidden_input3 = 128,256,346 -hidden_input4,hidden_input5,hidden_input6 = 480,560,686 +hidden_input, hidden_input2, hidden_input3 = 128, 256, 346 +hidden_input4, hidden_input5, hidden_input6 = 480, 560, 686 - -print('--------- Declare Hyper Parameters ----------') +print("--------- Declare Hyper Parameters ----------") # 2. Declare Weights -D_W1 = np.random.normal(size=(784,hidden_input),scale=(1. / np.sqrt(784 / 2.))) *0.002 +D_W1 = ( + np.random.normal(size=(784, hidden_input), scale=(1.0 / np.sqrt(784 / 2.0))) * 0.002 +) # D_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 D_b1 = np.zeros(hidden_input) -D_W2 = np.random.normal(size=(hidden_input,1),scale=(1. / np.sqrt(hidden_input / 2.))) *0.002 +D_W2 = ( + np.random.normal(size=(hidden_input, 1), scale=(1.0 / np.sqrt(hidden_input / 2.0))) + * 0.002 +) # D_b2 = np.random.normal(size=(1),scale=(1. / np.sqrt(1 / 2.))) *0.002 D_b2 = np.zeros(1) -G_W1 = np.random.normal(size=(G_input,hidden_input),scale=(1. / np.sqrt(G_input / 2.))) *0.002 +G_W1 = ( + np.random.normal(size=(G_input, hidden_input), scale=(1.0 / np.sqrt(G_input / 2.0))) + * 0.002 +) # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 G_b1 = np.zeros(hidden_input) -G_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.))) *0.002 +G_W2 = ( + np.random.normal( + size=(hidden_input, hidden_input2), scale=(1.0 / np.sqrt(hidden_input / 2.0)) + ) + * 0.002 +) # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 G_b2 = np.zeros(hidden_input2) -G_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.))) *0.002 +G_W3 = ( + np.random.normal( + size=(hidden_input2, hidden_input3), scale=(1.0 / np.sqrt(hidden_input2 / 2.0)) + ) + * 0.002 +) # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 G_b3 = np.zeros(hidden_input3) -G_W4 = np.random.normal(size=(hidden_input3,hidden_input4),scale=(1. / np.sqrt(hidden_input3 / 2.))) *0.002 +G_W4 = ( + np.random.normal( + size=(hidden_input3, hidden_input4), scale=(1.0 / np.sqrt(hidden_input3 / 2.0)) + ) + * 0.002 +) # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 G_b4 = np.zeros(hidden_input4) -G_W5 = np.random.normal(size=(hidden_input4,hidden_input5),scale=(1. / np.sqrt(hidden_input4 / 2.))) *0.002 +G_W5 = ( + np.random.normal( + size=(hidden_input4, hidden_input5), scale=(1.0 / np.sqrt(hidden_input4 / 2.0)) + ) + * 0.002 +) # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 G_b5 = np.zeros(hidden_input5) -G_W6 = np.random.normal(size=(hidden_input5,hidden_input6),scale=(1. / np.sqrt(hidden_input5 / 2.))) *0.002 +G_W6 = ( + np.random.normal( + size=(hidden_input5, hidden_input6), scale=(1.0 / np.sqrt(hidden_input5 / 2.0)) + ) + * 0.002 +) # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 G_b6 = np.zeros(hidden_input6) -G_W7 = np.random.normal(size=(hidden_input6,784),scale=(1. / np.sqrt(hidden_input6 / 2.))) *0.002 +G_W7 = ( + np.random.normal( + size=(hidden_input6, 784), scale=(1.0 / np.sqrt(hidden_input6 / 2.0)) + ) + * 0.002 +) # G_b2 = np.random.normal(size=(784),scale=(1. / np.sqrt(784 / 2.))) *0.002 G_b7 = np.zeros(784) # 3. For Adam Optimzier -v1,m1 = 0,0 -v2,m2 = 0,0 -v3,m3 = 0,0 -v4,m4 = 0,0 +v1, m1 = 0, 0 +v2, m2 = 0, 0 +v3, m3 = 0, 0 +v4, m4 = 0, 0 -v5,m5 = 0,0 -v6,m6 = 0,0 -v7,m7 = 0,0 -v8,m8 = 0,0 -v9,m9 = 0,0 -v10,m10 = 0,0 -v11,m11 = 0,0 -v12,m12 = 0,0 +v5, m5 = 0, 0 +v6, m6 = 0, 0 +v7, m7 = 0, 0 +v8, m8 = 0, 0 +v9, m9 = 0, 0 +v10, m10 = 0, 0 +v11, m11 = 0, 0 +v12, m12 = 0, 0 -v13,m13 = 0,0 -v14,m14 = 0,0 +v13, m13 = 0, 0 +v14, m14 = 0, 0 -v15,m15 = 0,0 -v16,m16 = 0,0 +v15, m15 = 0, 0 +v16, m16 = 0, 0 -v17,m17 = 0,0 -v18,m18 = 0,0 +v17, m17 = 0, 0 +v18, m18 = 0, 0 -beta_1,beta_2,eps = 0.9,0.999,0.00000001 +beta_1, beta_2, eps = 0.9, 0.999, 0.00000001 -print('--------- Started Training ----------') +print("--------- Started Training ----------") for iter in range(num_epoch): random_int = np.random.randint(len(images) - 5) - current_image = np.expand_dims(images[random_int],axis=0) + current_image = np.expand_dims(images[random_int], axis=0) # Func: Generate The first Fake Data - Z = np.random.uniform(-1., 1., size=[1, G_input]) + Z = np.random.uniform(-1.0, 1.0, size=[1, G_input]) Gl1 = Z.dot(G_W1) + G_b1 Gl1A = arctan(Gl1) Gl2 = Gl1A.dot(G_W2) + G_b2 @@ -164,38 +214,38 @@ for iter in range(num_epoch): Dl2_fA = log(Dl2_f) # Func: Cost D - D_cost = -np.log(Dl2_rA) + np.log(1.0- Dl2_fA) + D_cost = -np.log(Dl2_rA) + np.log(1.0 - Dl2_fA) # Func: Gradient - grad_f_w2_part_1 = 1/(1.0- Dl2_fA) - grad_f_w2_part_2 = d_log(Dl2_f) - grad_f_w2_part_3 = Dl1_fA - grad_f_w2 = grad_f_w2_part_3.T.dot(grad_f_w2_part_1 * grad_f_w2_part_2) + grad_f_w2_part_1 = 1 / (1.0 - Dl2_fA) + grad_f_w2_part_2 = d_log(Dl2_f) + grad_f_w2_part_3 = Dl1_fA + grad_f_w2 = grad_f_w2_part_3.T.dot(grad_f_w2_part_1 * grad_f_w2_part_2) grad_f_b2 = grad_f_w2_part_1 * grad_f_w2_part_2 - grad_f_w1_part_1 = (grad_f_w2_part_1 * grad_f_w2_part_2).dot(D_W2.T) - grad_f_w1_part_2 = d_ReLu(Dl1_f) - grad_f_w1_part_3 = current_fake_data - grad_f_w1 = grad_f_w1_part_3.T.dot(grad_f_w1_part_1 * grad_f_w1_part_2) - grad_f_b1 = grad_f_w1_part_1 * grad_f_w1_part_2 + grad_f_w1_part_1 = (grad_f_w2_part_1 * grad_f_w2_part_2).dot(D_W2.T) + grad_f_w1_part_2 = d_ReLu(Dl1_f) + grad_f_w1_part_3 = current_fake_data + grad_f_w1 = grad_f_w1_part_3.T.dot(grad_f_w1_part_1 * grad_f_w1_part_2) + grad_f_b1 = grad_f_w1_part_1 * grad_f_w1_part_2 - grad_r_w2_part_1 = - 1/Dl2_rA - grad_r_w2_part_2 = d_log(Dl2_r) - grad_r_w2_part_3 = Dl1_rA - grad_r_w2 = grad_r_w2_part_3.T.dot(grad_r_w2_part_1 * grad_r_w2_part_2) - grad_r_b2 = grad_r_w2_part_1 * grad_r_w2_part_2 + grad_r_w2_part_1 = -1 / Dl2_rA + grad_r_w2_part_2 = d_log(Dl2_r) + grad_r_w2_part_3 = Dl1_rA + grad_r_w2 = grad_r_w2_part_3.T.dot(grad_r_w2_part_1 * grad_r_w2_part_2) + grad_r_b2 = grad_r_w2_part_1 * grad_r_w2_part_2 - grad_r_w1_part_1 = (grad_r_w2_part_1 * grad_r_w2_part_2).dot(D_W2.T) - grad_r_w1_part_2 = d_ReLu(Dl1_r) - grad_r_w1_part_3 = current_image - grad_r_w1 = grad_r_w1_part_3.T.dot(grad_r_w1_part_1 * grad_r_w1_part_2) - grad_r_b1 = grad_r_w1_part_1 * grad_r_w1_part_2 + grad_r_w1_part_1 = (grad_r_w2_part_1 * grad_r_w2_part_2).dot(D_W2.T) + grad_r_w1_part_2 = d_ReLu(Dl1_r) + grad_r_w1_part_3 = current_image + grad_r_w1 = grad_r_w1_part_3.T.dot(grad_r_w1_part_1 * grad_r_w1_part_2) + grad_r_b1 = grad_r_w1_part_1 * grad_r_w1_part_2 - grad_w1 =grad_f_w1 + grad_r_w1 - grad_b1 =grad_f_b1 + grad_r_b1 + grad_w1 = grad_f_w1 + grad_r_w1 + grad_b1 = grad_f_b1 + grad_r_b1 - grad_w2 =grad_f_w2 + grad_r_w2 - grad_b2 =grad_f_b2 + grad_r_b2 + grad_w2 = grad_f_w2 + grad_r_w2 + grad_b2 = grad_f_b2 + grad_r_b2 # ---- Update Gradient ---- m1 = beta_1 * m1 + (1 - beta_1) * grad_w1 @@ -210,14 +260,22 @@ for iter in range(num_epoch): m4 = beta_1 * m4 + (1 - beta_1) * grad_b2 v4 = beta_2 * v4 + (1 - beta_2) * grad_b2 ** 2 - D_W1 = D_W1 - (learing_rate / (np.sqrt(v1 /(1-beta_2) ) + eps)) * (m1/(1-beta_1)) - D_b1 = D_b1 - (learing_rate / (np.sqrt(v2 /(1-beta_2) ) + eps)) * (m2/(1-beta_1)) + D_W1 = D_W1 - (learing_rate / (np.sqrt(v1 / (1 - beta_2)) + eps)) * ( + m1 / (1 - beta_1) + ) + D_b1 = D_b1 - (learing_rate / (np.sqrt(v2 / (1 - beta_2)) + eps)) * ( + m2 / (1 - beta_1) + ) - D_W2 = D_W2 - (learing_rate / (np.sqrt(v3 /(1-beta_2) ) + eps)) * (m3/(1-beta_1)) - D_b2 = D_b2 - (learing_rate / (np.sqrt(v4 /(1-beta_2) ) + eps)) * (m4/(1-beta_1)) + D_W2 = D_W2 - (learing_rate / (np.sqrt(v3 / (1 - beta_2)) + eps)) * ( + m3 / (1 - beta_1) + ) + D_b2 = D_b2 - (learing_rate / (np.sqrt(v4 / (1 - beta_2)) + eps)) * ( + m4 / (1 - beta_1) + ) # Func: Forward Feed for G - Z = np.random.uniform(-1., 1., size=[1, G_input]) + Z = np.random.uniform(-1.0, 1.0, size=[1, G_input]) Gl1 = Z.dot(G_W1) + G_b1 Gl1A = arctan(Gl1) Gl2 = Gl1A.dot(G_W2) + G_b2 @@ -244,7 +302,9 @@ for iter in range(num_epoch): G_cost = -np.log(Dl2_A) # Func: Gradient - grad_G_w7_part_1 = ((-1/Dl2_A) * d_log(Dl2).dot(D_W2.T) * (d_ReLu(Dl1))).dot(D_W1.T) + grad_G_w7_part_1 = ((-1 / Dl2_A) * d_log(Dl2).dot(D_W2.T) * (d_ReLu(Dl1))).dot( + D_W1.T + ) grad_G_w7_part_2 = d_log(Gl7) grad_G_w7_part_3 = Gl6A grad_G_w7 = grad_G_w7_part_3.T.dot(grad_G_w7_part_1 * grad_G_w7_part_1) @@ -254,31 +314,31 @@ for iter in range(num_epoch): grad_G_w6_part_2 = d_ReLu(Gl6) grad_G_w6_part_3 = Gl5A grad_G_w6 = grad_G_w6_part_3.T.dot(grad_G_w6_part_1 * grad_G_w6_part_2) - grad_G_b6 = (grad_G_w6_part_1 * grad_G_w6_part_2) + grad_G_b6 = grad_G_w6_part_1 * grad_G_w6_part_2 grad_G_w5_part_1 = (grad_G_w6_part_1 * grad_G_w6_part_2).dot(G_W6.T) grad_G_w5_part_2 = d_tanh(Gl5) grad_G_w5_part_3 = Gl4A grad_G_w5 = grad_G_w5_part_3.T.dot(grad_G_w5_part_1 * grad_G_w5_part_2) - grad_G_b5 = (grad_G_w5_part_1 * grad_G_w5_part_2) + grad_G_b5 = grad_G_w5_part_1 * grad_G_w5_part_2 grad_G_w4_part_1 = (grad_G_w5_part_1 * grad_G_w5_part_2).dot(G_W5.T) grad_G_w4_part_2 = d_ReLu(Gl4) grad_G_w4_part_3 = Gl3A grad_G_w4 = grad_G_w4_part_3.T.dot(grad_G_w4_part_1 * grad_G_w4_part_2) - grad_G_b4 = (grad_G_w4_part_1 * grad_G_w4_part_2) + grad_G_b4 = grad_G_w4_part_1 * grad_G_w4_part_2 grad_G_w3_part_1 = (grad_G_w4_part_1 * grad_G_w4_part_2).dot(G_W4.T) grad_G_w3_part_2 = d_arctan(Gl3) grad_G_w3_part_3 = Gl2A grad_G_w3 = grad_G_w3_part_3.T.dot(grad_G_w3_part_1 * grad_G_w3_part_2) - grad_G_b3 = (grad_G_w3_part_1 * grad_G_w3_part_2) + grad_G_b3 = grad_G_w3_part_1 * grad_G_w3_part_2 grad_G_w2_part_1 = (grad_G_w3_part_1 * grad_G_w3_part_2).dot(G_W3.T) grad_G_w2_part_2 = d_ReLu(Gl2) grad_G_w2_part_3 = Gl1A grad_G_w2 = grad_G_w2_part_3.T.dot(grad_G_w2_part_1 * grad_G_w2_part_2) - grad_G_b2 = (grad_G_w2_part_1 * grad_G_w2_part_2) + grad_G_b2 = grad_G_w2_part_1 * grad_G_w2_part_2 grad_G_w1_part_1 = (grad_G_w2_part_1 * grad_G_w2_part_2).dot(G_W2.T) grad_G_w1_part_2 = d_arctan(Gl1) @@ -329,29 +389,57 @@ for iter in range(num_epoch): m18 = beta_1 * m18 + (1 - beta_1) * grad_G_b7 v18 = beta_2 * v18 + (1 - beta_2) * grad_G_b7 ** 2 - G_W1 = G_W1 - (learing_rate / (np.sqrt(v5 /(1-beta_2) ) + eps)) * (m5/(1-beta_1)) - G_b1 = G_b1 - (learing_rate / (np.sqrt(v6 /(1-beta_2) ) + eps)) * (m6/(1-beta_1)) + G_W1 = G_W1 - (learing_rate / (np.sqrt(v5 / (1 - beta_2)) + eps)) * ( + m5 / (1 - beta_1) + ) + G_b1 = G_b1 - (learing_rate / (np.sqrt(v6 / (1 - beta_2)) + eps)) * ( + m6 / (1 - beta_1) + ) - G_W2 = G_W2 - (learing_rate / (np.sqrt(v7 /(1-beta_2) ) + eps)) * (m7/(1-beta_1)) - G_b2 = G_b2 - (learing_rate / (np.sqrt(v8 /(1-beta_2) ) + eps)) * (m8/(1-beta_1)) + G_W2 = G_W2 - (learing_rate / (np.sqrt(v7 / (1 - beta_2)) + eps)) * ( + m7 / (1 - beta_1) + ) + G_b2 = G_b2 - (learing_rate / (np.sqrt(v8 / (1 - beta_2)) + eps)) * ( + m8 / (1 - beta_1) + ) - G_W3 = G_W3 - (learing_rate / (np.sqrt(v9 /(1-beta_2) ) + eps)) * (m9/(1-beta_1)) - G_b3 = G_b3 - (learing_rate / (np.sqrt(v10 /(1-beta_2) ) + eps)) * (m10/(1-beta_1)) + G_W3 = G_W3 - (learing_rate / (np.sqrt(v9 / (1 - beta_2)) + eps)) * ( + m9 / (1 - beta_1) + ) + G_b3 = G_b3 - (learing_rate / (np.sqrt(v10 / (1 - beta_2)) + eps)) * ( + m10 / (1 - beta_1) + ) - G_W4 = G_W4 - (learing_rate / (np.sqrt(v11 /(1-beta_2) ) + eps)) * (m11/(1-beta_1)) - G_b4 = G_b4 - (learing_rate / (np.sqrt(v12 /(1-beta_2) ) + eps)) * (m12/(1-beta_1)) + G_W4 = G_W4 - (learing_rate / (np.sqrt(v11 / (1 - beta_2)) + eps)) * ( + m11 / (1 - beta_1) + ) + G_b4 = G_b4 - (learing_rate / (np.sqrt(v12 / (1 - beta_2)) + eps)) * ( + m12 / (1 - beta_1) + ) - G_W5 = G_W5 - (learing_rate / (np.sqrt(v13 /(1-beta_2) ) + eps)) * (m13/(1-beta_1)) - G_b5 = G_b5 - (learing_rate / (np.sqrt(v14 /(1-beta_2) ) + eps)) * (m14/(1-beta_1)) + G_W5 = G_W5 - (learing_rate / (np.sqrt(v13 / (1 - beta_2)) + eps)) * ( + m13 / (1 - beta_1) + ) + G_b5 = G_b5 - (learing_rate / (np.sqrt(v14 / (1 - beta_2)) + eps)) * ( + m14 / (1 - beta_1) + ) - G_W6 = G_W6 - (learing_rate / (np.sqrt(v15 /(1-beta_2) ) + eps)) * (m15/(1-beta_1)) - G_b6 = G_b6 - (learing_rate / (np.sqrt(v16 /(1-beta_2) ) + eps)) * (m16/(1-beta_1)) + G_W6 = G_W6 - (learing_rate / (np.sqrt(v15 / (1 - beta_2)) + eps)) * ( + m15 / (1 - beta_1) + ) + G_b6 = G_b6 - (learing_rate / (np.sqrt(v16 / (1 - beta_2)) + eps)) * ( + m16 / (1 - beta_1) + ) - G_W7 = G_W7 - (learing_rate / (np.sqrt(v17 /(1-beta_2) ) + eps)) * (m17/(1-beta_1)) - G_b7 = G_b7 - (learing_rate / (np.sqrt(v18 /(1-beta_2) ) + eps)) * (m18/(1-beta_1)) + G_W7 = G_W7 - (learing_rate / (np.sqrt(v17 / (1 - beta_2)) + eps)) * ( + m17 / (1 - beta_1) + ) + G_b7 = G_b7 - (learing_rate / (np.sqrt(v18 / (1 - beta_2)) + eps)) * ( + m18 / (1 - beta_1) + ) # --- Print Error ---- - #print("Current Iter: ",iter, " Current D cost:",D_cost, " Current G cost: ", G_cost,end='\r') + # print("Current Iter: ",iter, " Current D cost:",D_cost, " Current G cost: ", G_cost,end='\r') if iter == 0: learing_rate = learing_rate * 0.01 @@ -359,12 +447,20 @@ for iter in range(num_epoch): learing_rate = learing_rate * 0.01 # ---- Print to Out put ---- - if iter%10 == 0: + if iter % 10 == 0: - print("Current Iter: ",iter, " Current D cost:",D_cost, " Current G cost: ", G_cost,end='\r') - print('--------- Show Example Result See Tab Above ----------') - print('--------- Wait for the image to load ---------') - Z = np.random.uniform(-1., 1., size=[16, G_input]) + print( + "Current Iter: ", + iter, + " Current D cost:", + D_cost, + " Current G cost: ", + G_cost, + end="\r", + ) + print("--------- Show Example Result See Tab Above ----------") + print("--------- Wait for the image to load ---------") + Z = np.random.uniform(-1.0, 1.0, size=[16, G_input]) Gl1 = Z.dot(G_W1) + G_b1 Gl1A = arctan(Gl1) @@ -384,8 +480,19 @@ for iter in range(num_epoch): current_fake_data = log(Gl7) fig = plot(current_fake_data) - fig.savefig('Click_Me_{}.png'.format(str(iter).zfill(3)+"_Ginput_"+str(G_input)+ \ - "_hiddenone"+str(hidden_input) + "_hiddentwo"+str(hidden_input2) + "_LR_" + str(learing_rate) - ), bbox_inches='tight') -#for complete explanation visit https://towardsdatascience.com/only-numpy-implementing-gan-general-adversarial-networks-and-adam-optimizer-using-numpy-with-2a7e4e032021 + fig.savefig( + "Click_Me_{}.png".format( + str(iter).zfill(3) + + "_Ginput_" + + str(G_input) + + "_hiddenone" + + str(hidden_input) + + "_hiddentwo" + + str(hidden_input2) + + "_LR_" + + str(learing_rate) + ), + bbox_inches="tight", + ) +# for complete explanation visit https://towardsdatascience.com/only-numpy-implementing-gan-general-adversarial-networks-and-adam-optimizer-using-numpy-with-2a7e4e032021 # -- end code -- diff --git a/neural_network/input_data.py b/neural_network/input_data.py index 983063f0b..5e6c433aa 100644 --- a/neural_network/input_data.py +++ b/neural_network/input_data.py @@ -34,20 +34,20 @@ from tensorflow.python.framework import random_seed from tensorflow.python.platform import gfile from tensorflow.python.util.deprecation import deprecated -_Datasets = collections.namedtuple('_Datasets', ['train', 'validation', 'test']) +_Datasets = collections.namedtuple("_Datasets", ["train", "validation", "test"]) # CVDF mirror of http://yann.lecun.com/exdb/mnist/ -DEFAULT_SOURCE_URL = 'https://storage.googleapis.com/cvdf-datasets/mnist/' +DEFAULT_SOURCE_URL = "https://storage.googleapis.com/cvdf-datasets/mnist/" def _read32(bytestream): - dt = numpy.dtype(numpy.uint32).newbyteorder('>') - return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] + dt = numpy.dtype(numpy.uint32).newbyteorder(">") + return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] -@deprecated(None, 'Please use tf.data to implement this functionality.') +@deprecated(None, "Please use tf.data to implement this functionality.") def _extract_images(f): - """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. + """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. Args: f: A file object that can be passed into a gzip reader. @@ -59,34 +59,35 @@ def _extract_images(f): ValueError: If the bytestream does not start with 2051. """ - print('Extracting', f.name) - with gzip.GzipFile(fileobj=f) as bytestream: - magic = _read32(bytestream) - if magic != 2051: - raise ValueError('Invalid magic number %d in MNIST image file: %s' % - (magic, f.name)) - num_images = _read32(bytestream) - rows = _read32(bytestream) - cols = _read32(bytestream) - buf = bytestream.read(rows * cols * num_images) - data = numpy.frombuffer(buf, dtype=numpy.uint8) - data = data.reshape(num_images, rows, cols, 1) - return data + print("Extracting", f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2051: + raise ValueError( + "Invalid magic number %d in MNIST image file: %s" % (magic, f.name) + ) + num_images = _read32(bytestream) + rows = _read32(bytestream) + cols = _read32(bytestream) + buf = bytestream.read(rows * cols * num_images) + data = numpy.frombuffer(buf, dtype=numpy.uint8) + data = data.reshape(num_images, rows, cols, 1) + return data -@deprecated(None, 'Please use tf.one_hot on tensors.') +@deprecated(None, "Please use tf.one_hot on tensors.") def _dense_to_one_hot(labels_dense, num_classes): - """Convert class labels from scalars to one-hot vectors.""" - num_labels = labels_dense.shape[0] - index_offset = numpy.arange(num_labels) * num_classes - labels_one_hot = numpy.zeros((num_labels, num_classes)) - labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1 - return labels_one_hot + """Convert class labels from scalars to one-hot vectors.""" + num_labels = labels_dense.shape[0] + index_offset = numpy.arange(num_labels) * num_classes + labels_one_hot = numpy.zeros((num_labels, num_classes)) + labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1 + return labels_one_hot -@deprecated(None, 'Please use tf.data to implement this functionality.') +@deprecated(None, "Please use tf.data to implement this functionality.") def _extract_labels(f, one_hot=False, num_classes=10): - """Extract the labels into a 1D uint8 numpy array [index]. + """Extract the labels into a 1D uint8 numpy array [index]. Args: f: A file object that can be passed into a gzip reader. @@ -99,37 +100,43 @@ def _extract_labels(f, one_hot=False, num_classes=10): Raises: ValueError: If the bystream doesn't start with 2049. """ - print('Extracting', f.name) - with gzip.GzipFile(fileobj=f) as bytestream: - magic = _read32(bytestream) - if magic != 2049: - raise ValueError('Invalid magic number %d in MNIST label file: %s' % - (magic, f.name)) - num_items = _read32(bytestream) - buf = bytestream.read(num_items) - labels = numpy.frombuffer(buf, dtype=numpy.uint8) - if one_hot: - return _dense_to_one_hot(labels, num_classes) - return labels + print("Extracting", f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2049: + raise ValueError( + "Invalid magic number %d in MNIST label file: %s" % (magic, f.name) + ) + num_items = _read32(bytestream) + buf = bytestream.read(num_items) + labels = numpy.frombuffer(buf, dtype=numpy.uint8) + if one_hot: + return _dense_to_one_hot(labels, num_classes) + return labels class _DataSet(object): - """Container class for a _DataSet (deprecated). + """Container class for a _DataSet (deprecated). THIS CLASS IS DEPRECATED. """ - @deprecated(None, 'Please use alternatives such as official/mnist/_DataSet.py' - ' from tensorflow/models.') - def __init__(self, - images, - labels, - fake_data=False, - one_hot=False, - dtype=dtypes.float32, - reshape=True, - seed=None): - """Construct a _DataSet. + @deprecated( + None, + "Please use alternatives such as official/mnist/_DataSet.py" + " from tensorflow/models.", + ) + def __init__( + self, + images, + labels, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + seed=None, + ): + """Construct a _DataSet. one_hot arg is used only if fake_data is true. `dtype` can be either `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into @@ -146,101 +153,105 @@ class _DataSet(object): reshape: Bool. If True returned images are returned flattened to vectors. seed: The random seed to use. """ - seed1, seed2 = random_seed.get_seed(seed) - # If op level seed is not set, use whatever graph level seed is returned - numpy.random.seed(seed1 if seed is None else seed2) - dtype = dtypes.as_dtype(dtype).base_dtype - if dtype not in (dtypes.uint8, dtypes.float32): - raise TypeError('Invalid image dtype %r, expected uint8 or float32' % - dtype) - if fake_data: - self._num_examples = 10000 - self.one_hot = one_hot - else: - assert images.shape[0] == labels.shape[0], ( - 'images.shape: %s labels.shape: %s' % (images.shape, labels.shape)) - self._num_examples = images.shape[0] + seed1, seed2 = random_seed.get_seed(seed) + # If op level seed is not set, use whatever graph level seed is returned + numpy.random.seed(seed1 if seed is None else seed2) + dtype = dtypes.as_dtype(dtype).base_dtype + if dtype not in (dtypes.uint8, dtypes.float32): + raise TypeError("Invalid image dtype %r, expected uint8 or float32" % dtype) + if fake_data: + self._num_examples = 10000 + self.one_hot = one_hot + else: + assert ( + images.shape[0] == labels.shape[0] + ), "images.shape: %s labels.shape: %s" % (images.shape, labels.shape) + self._num_examples = images.shape[0] - # Convert shape from [num examples, rows, columns, depth] - # to [num examples, rows*columns] (assuming depth == 1) - if reshape: - assert images.shape[3] == 1 - images = images.reshape(images.shape[0], - images.shape[1] * images.shape[2]) - if dtype == dtypes.float32: - # Convert from [0, 255] -> [0.0, 1.0]. - images = images.astype(numpy.float32) - images = numpy.multiply(images, 1.0 / 255.0) - self._images = images - self._labels = labels - self._epochs_completed = 0 - self._index_in_epoch = 0 + # Convert shape from [num examples, rows, columns, depth] + # to [num examples, rows*columns] (assuming depth == 1) + if reshape: + assert images.shape[3] == 1 + images = images.reshape( + images.shape[0], images.shape[1] * images.shape[2] + ) + if dtype == dtypes.float32: + # Convert from [0, 255] -> [0.0, 1.0]. + images = images.astype(numpy.float32) + images = numpy.multiply(images, 1.0 / 255.0) + self._images = images + self._labels = labels + self._epochs_completed = 0 + self._index_in_epoch = 0 - @property - def images(self): - return self._images + @property + def images(self): + return self._images - @property - def labels(self): - return self._labels + @property + def labels(self): + return self._labels - @property - def num_examples(self): - return self._num_examples + @property + def num_examples(self): + return self._num_examples - @property - def epochs_completed(self): - return self._epochs_completed + @property + def epochs_completed(self): + return self._epochs_completed - def next_batch(self, batch_size, fake_data=False, shuffle=True): - """Return the next `batch_size` examples from this data set.""" - if fake_data: - fake_image = [1] * 784 - if self.one_hot: - fake_label = [1] + [0] * 9 - else: - fake_label = 0 - return [fake_image for _ in xrange(batch_size) - ], [fake_label for _ in xrange(batch_size)] - start = self._index_in_epoch - # Shuffle for the first epoch - if self._epochs_completed == 0 and start == 0 and shuffle: - perm0 = numpy.arange(self._num_examples) - numpy.random.shuffle(perm0) - self._images = self.images[perm0] - self._labels = self.labels[perm0] - # Go to the next epoch - if start + batch_size > self._num_examples: - # Finished epoch - self._epochs_completed += 1 - # Get the rest examples in this epoch - rest_num_examples = self._num_examples - start - images_rest_part = self._images[start:self._num_examples] - labels_rest_part = self._labels[start:self._num_examples] - # Shuffle the data - if shuffle: - perm = numpy.arange(self._num_examples) - numpy.random.shuffle(perm) - self._images = self.images[perm] - self._labels = self.labels[perm] - # Start next epoch - start = 0 - self._index_in_epoch = batch_size - rest_num_examples - end = self._index_in_epoch - images_new_part = self._images[start:end] - labels_new_part = self._labels[start:end] - return numpy.concatenate((images_rest_part, images_new_part), - axis=0), numpy.concatenate( - (labels_rest_part, labels_new_part), axis=0) - else: - self._index_in_epoch += batch_size - end = self._index_in_epoch - return self._images[start:end], self._labels[start:end] + def next_batch(self, batch_size, fake_data=False, shuffle=True): + """Return the next `batch_size` examples from this data set.""" + if fake_data: + fake_image = [1] * 784 + if self.one_hot: + fake_label = [1] + [0] * 9 + else: + fake_label = 0 + return ( + [fake_image for _ in xrange(batch_size)], + [fake_label for _ in xrange(batch_size)], + ) + start = self._index_in_epoch + # Shuffle for the first epoch + if self._epochs_completed == 0 and start == 0 and shuffle: + perm0 = numpy.arange(self._num_examples) + numpy.random.shuffle(perm0) + self._images = self.images[perm0] + self._labels = self.labels[perm0] + # Go to the next epoch + if start + batch_size > self._num_examples: + # Finished epoch + self._epochs_completed += 1 + # Get the rest examples in this epoch + rest_num_examples = self._num_examples - start + images_rest_part = self._images[start : self._num_examples] + labels_rest_part = self._labels[start : self._num_examples] + # Shuffle the data + if shuffle: + perm = numpy.arange(self._num_examples) + numpy.random.shuffle(perm) + self._images = self.images[perm] + self._labels = self.labels[perm] + # Start next epoch + start = 0 + self._index_in_epoch = batch_size - rest_num_examples + end = self._index_in_epoch + images_new_part = self._images[start:end] + labels_new_part = self._labels[start:end] + return ( + numpy.concatenate((images_rest_part, images_new_part), axis=0), + numpy.concatenate((labels_rest_part, labels_new_part), axis=0), + ) + else: + self._index_in_epoch += batch_size + end = self._index_in_epoch + return self._images[start:end], self._labels[start:end] -@deprecated(None, 'Please write your own downloading logic.') +@deprecated(None, "Please write your own downloading logic.") def _maybe_download(filename, work_directory, source_url): - """Download the data from source url, unless it's already here. + """Download the data from source url, unless it's already here. Args: filename: string, name of the file in the directory. @@ -250,83 +261,90 @@ def _maybe_download(filename, work_directory, source_url): Returns: Path to resulting file. """ - if not gfile.Exists(work_directory): - gfile.MakeDirs(work_directory) - filepath = os.path.join(work_directory, filename) - if not gfile.Exists(filepath): - urllib.request.urlretrieve(source_url, filepath) - with gfile.GFile(filepath) as f: - size = f.size() - print('Successfully downloaded', filename, size, 'bytes.') - return filepath + if not gfile.Exists(work_directory): + gfile.MakeDirs(work_directory) + filepath = os.path.join(work_directory, filename) + if not gfile.Exists(filepath): + urllib.request.urlretrieve(source_url, filepath) + with gfile.GFile(filepath) as f: + size = f.size() + print("Successfully downloaded", filename, size, "bytes.") + return filepath -@deprecated(None, 'Please use alternatives such as:' - ' tensorflow_datasets.load(\'mnist\')') -def read_data_sets(train_dir, - fake_data=False, - one_hot=False, - dtype=dtypes.float32, - reshape=True, - validation_size=5000, - seed=None, - source_url=DEFAULT_SOURCE_URL): - if fake_data: +@deprecated( + None, "Please use alternatives such as:" " tensorflow_datasets.load('mnist')" +) +def read_data_sets( + train_dir, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + validation_size=5000, + seed=None, + source_url=DEFAULT_SOURCE_URL, +): + if fake_data: - def fake(): - return _DataSet([], [], - fake_data=True, - one_hot=one_hot, - dtype=dtype, - seed=seed) + def fake(): + return _DataSet( + [], [], fake_data=True, one_hot=one_hot, dtype=dtype, seed=seed + ) + + train = fake() + validation = fake() + test = fake() + return _Datasets(train=train, validation=validation, test=test) + + if not source_url: # empty string check + source_url = DEFAULT_SOURCE_URL + + train_images_file = "train-images-idx3-ubyte.gz" + train_labels_file = "train-labels-idx1-ubyte.gz" + test_images_file = "t10k-images-idx3-ubyte.gz" + test_labels_file = "t10k-labels-idx1-ubyte.gz" + + local_file = _maybe_download( + train_images_file, train_dir, source_url + train_images_file + ) + with gfile.Open(local_file, "rb") as f: + train_images = _extract_images(f) + + local_file = _maybe_download( + train_labels_file, train_dir, source_url + train_labels_file + ) + with gfile.Open(local_file, "rb") as f: + train_labels = _extract_labels(f, one_hot=one_hot) + + local_file = _maybe_download( + test_images_file, train_dir, source_url + test_images_file + ) + with gfile.Open(local_file, "rb") as f: + test_images = _extract_images(f) + + local_file = _maybe_download( + test_labels_file, train_dir, source_url + test_labels_file + ) + with gfile.Open(local_file, "rb") as f: + test_labels = _extract_labels(f, one_hot=one_hot) + + if not 0 <= validation_size <= len(train_images): + raise ValueError( + "Validation size should be between 0 and {}. Received: {}.".format( + len(train_images), validation_size + ) + ) + + validation_images = train_images[:validation_size] + validation_labels = train_labels[:validation_size] + train_images = train_images[validation_size:] + train_labels = train_labels[validation_size:] + + options = dict(dtype=dtype, reshape=reshape, seed=seed) + + train = _DataSet(train_images, train_labels, **options) + validation = _DataSet(validation_images, validation_labels, **options) + test = _DataSet(test_images, test_labels, **options) - train = fake() - validation = fake() - test = fake() return _Datasets(train=train, validation=validation, test=test) - - if not source_url: # empty string check - source_url = DEFAULT_SOURCE_URL - - train_images_file = 'train-images-idx3-ubyte.gz' - train_labels_file = 'train-labels-idx1-ubyte.gz' - test_images_file = 't10k-images-idx3-ubyte.gz' - test_labels_file = 't10k-labels-idx1-ubyte.gz' - - local_file = _maybe_download(train_images_file, train_dir, - source_url + train_images_file) - with gfile.Open(local_file, 'rb') as f: - train_images = _extract_images(f) - - local_file = _maybe_download(train_labels_file, train_dir, - source_url + train_labels_file) - with gfile.Open(local_file, 'rb') as f: - train_labels = _extract_labels(f, one_hot=one_hot) - - local_file = _maybe_download(test_images_file, train_dir, - source_url + test_images_file) - with gfile.Open(local_file, 'rb') as f: - test_images = _extract_images(f) - - local_file = _maybe_download(test_labels_file, train_dir, - source_url + test_labels_file) - with gfile.Open(local_file, 'rb') as f: - test_labels = _extract_labels(f, one_hot=one_hot) - - if not 0 <= validation_size <= len(train_images): - raise ValueError( - 'Validation size should be between 0 and {}. Received: {}.'.format( - len(train_images), validation_size)) - - validation_images = train_images[:validation_size] - validation_labels = train_labels[:validation_size] - train_images = train_images[validation_size:] - train_labels = train_labels[validation_size:] - - options = dict(dtype=dtype, reshape=reshape, seed=seed) - - train = _DataSet(train_images, train_labels, **options) - validation = _DataSet(validation_images, validation_labels, **options) - test = _DataSet(test_images, test_labels, **options) - - return _Datasets(train=train, validation=validation, test=test) diff --git a/other/least_recently_used.py b/other/least_recently_used.py index 2932e9c18..e1b5ab5bd 100644 --- a/other/least_recently_used.py +++ b/other/least_recently_used.py @@ -2,12 +2,13 @@ from abc import abstractmethod import sys from collections import deque + class LRUCache: """ Page Replacement Algorithm, Least Recently Used (LRU) Caching.""" - dq_store = object() # Cache store of keys - key_reference_map = object() # References of the keys in cache - _MAX_CAPACITY: int = 10 # Maximum capacity of cache + dq_store = object() # Cache store of keys + key_reference_map = object() # References of the keys in cache + _MAX_CAPACITY: int = 10 # Maximum capacity of cache @abstractmethod def __init__(self, n: int): @@ -19,7 +20,7 @@ class LRUCache: if not n: LRUCache._MAX_CAPACITY = sys.maxsize elif n < 0: - raise ValueError('n should be an integer greater than 0.') + raise ValueError("n should be an integer greater than 0.") else: LRUCache._MAX_CAPACITY = n @@ -51,6 +52,7 @@ class LRUCache: for k in self.dq_store: print(k) + if __name__ == "__main__": lru_cache = LRUCache(4) lru_cache.refer(1) diff --git a/project_euler/problem_20/sol4.py b/project_euler/problem_20/sol4.py index 50ebca5a0..4c597220f 100644 --- a/project_euler/problem_20/sol4.py +++ b/project_euler/problem_20/sol4.py @@ -27,7 +27,7 @@ def solution(n): """ fact = 1 result = 0 - for i in range(1,n + 1): + for i in range(1, n + 1): fact *= i for j in str(fact): diff --git a/project_euler/problem_99/sol1.py b/project_euler/problem_99/sol1.py index 6729927df..713bf65ca 100644 --- a/project_euler/problem_99/sol1.py +++ b/project_euler/problem_99/sol1.py @@ -14,15 +14,13 @@ import os from math import log10 -def find_largest(data_file: str="base_exp.txt") -> int: +def find_largest(data_file: str = "base_exp.txt") -> int: """ >>> find_largest() 709 """ largest = [0, 0] - for i, line in enumerate( - open(os.path.join(os.path.dirname(__file__), data_file)) - ): + for i, line in enumerate(open(os.path.join(os.path.dirname(__file__), data_file))): a, x = list(map(int, line.split(","))) if x * log10(a) > largest[0]: largest = [x * log10(a), i + 1] diff --git a/web_programming/get_imdbtop.py b/web_programming/get_imdbtop.py index 95fbeba7a..522e423b4 100644 --- a/web_programming/get_imdbtop.py +++ b/web_programming/get_imdbtop.py @@ -3,8 +3,10 @@ import requests def imdb_top(imdb_top_n): - base_url = (f"https://www.imdb.com/search/title?title_type=" - f"feature&sort=num_votes,desc&count={imdb_top_n}") + base_url = ( + f"https://www.imdb.com/search/title?title_type=" + f"feature&sort=num_votes,desc&count={imdb_top_n}" + ) source = BeautifulSoup(requests.get(base_url).content, "html.parser") for m in source.findAll("div", class_="lister-item mode-advanced"): print("\n" + m.h3.a.text) # movie's name