0.4.3 NN - Autograd - Random - Plot - Faster iteration (#42)

* testing yielding over iterators

* tri iterator

* iter complete

* sorting for tensors

* matrix exponential using pade

* neural network basics

* remove req

* move images

* full implementation of autograd and better networks

* Added plplot bindings

* merge PRs

* docs

* remove dataframes

* more activation

* default bias

* random creation methods using alea

Author: christopherzimmerman

README.md

@@ -187,61 +187,70 @@ puts a.matmul(a)
 #  [15, 22]]
 ```
 
-### DataFrames
+### Machine Learning
 
-For more structured data, consider using a `DataFrame`
+`Num::Grad` provides a pure-crystal approach to find derivatives of
+mathematical functions.  Use a `Num::Grad::Variable` with a `Num::Grad::Context`
+to easily compute these derivatives.
 
 ```crystal
-df = DataFrame.from_items(
-  foo: [1, 2, 3, 4, 5].to_tensor,
-  bar: [2.73, 3.1, 4.8, 5.1, 3.2],
-)
-
-puts df
-
-#    foo   bar
-# 0    1  2.73
-# 1    2   3.1
-# 2    3   4.8
-# 3    4   5.1
-# 4    5   3.2
+ctx = Num::Grad::Context(Tensor(Float64)).new
+
+x = ctx.variable([3.0])
+y = ctx.variable([2.0])
+
+# f(x) = x ** y
+f = x ** y
+puts f # => [9]
+
+f.backprop
+
+# df/dx = y * x = 6.0
+puts x.grad # => [6.0]
 ```
 
-A `DataFrame` maintains types while still providing convenient
-mapping and reduction operations
+`Num::NN` contains an extension to `Num::Grad` that provides an easy-to-use
+interface to assist in creating neural networks.  Designing and creating
+a network is simple using Crystal's block syntax.
 
 ```crystal
-puts df.c[:foo]
+ctx = Num::Grad::Context(Tensor(Float64)).new
 
-# 0  1
-# 1  2
-# 2  3
-# 3  4
-# 4  5
-# Name: foo
-# dtype: Int32
+x_train = [[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]].to_tensor
+y_train = [[0.0], [1.0], [1.0], [0.0]].to_tensor
 
-puts typeof(df.c[:foo])
+x = ctx.variable(x_train)
 
-# Series(Int32, Int32)
+net = Num::NN::Network.new(ctx) do
 
-puts df.sum
+  # A basic network with a single hidden layer using
+  # a ReLU activation function
+  linear(2, 3)
+  relu
+  linear(3, 1)
 
-# foo     15
-# bar  18.93
-```
+  # SGD Optimizer
+  sgd 0.7
 
-With operations that broadcast across the `DataFrame`
+  # Sigmoid Cross Entropy to calculate loss
+  sigmoid_cross_entropy_loss
+end
 
-```crystal
-puts df.greater(df.mean)
-
-#      foo    bar
-# 0  false  false
-# 1  false  false
-# 2  false   true
-# 3   true   true
-# 4   true  false
+500.times do |epoch|
+  y_pred = net.forward(x)
+  loss = net.loss(y_pred, y_train)
+  puts "Epoch: #{epoch} - Loss #{loss}"
+  loss.backprop
+  net.optimizer.update
+end
+
+# Clip results to make a prediction
+puts net.forward(x).value.map { |el| el > 0 ? 1 : 0}
+
+# [[0],
+#  [1],
+#  [1],
+#  [0]]
 ```
 
 Review the documentation for full implementation details, and if something is missing,

examples/basic_xor_classifier/README.md

@@ -0,0 +1,74 @@
+## Basic XOR Classifier
+
+The following implements a simple XOR classifier to show how to use
+`num.cr`'s `Network` class.  Plotting is done via `ishi`.
+
+```crystal
+ctx = Num::Grad::Context(Tensor(Float64)).new
+
+bsz = 32
+
+x_train_bool = Tensor.random(0_u8...2_u8, [bsz * 100, 2])
+
+y_bool = x_train_bool[..., ...1] ^ x_train_bool[..., 1...]
+
+x_train = ctx.variable(x_train_bool.as_type(Float64))
+y = y_bool.as_type(Float64)
+
+net = Num::NN::Network.new(ctx) do
+  linear 2, 3
+  relu
+  linear 3, 1
+  sgd 0.7
+  sigmoid_cross_entropy_loss
+end
+
+losses = [] of Float64
+
+50.times do |epoch|
+  100.times do |batch_id|
+    offset = batch_id * 32
+    x = x_train[offset...offset + 32]
+    target = y[offset...offset + 32]
+
+    y_pred = net.forward(x)
+
+    loss = net.loss(y_pred, target)
+
+    puts "Epoch is: #{epoch}"
+    puts "Batch id: #{batch_id}"
+    puts "Loss is: #{loss.value.value}"
+    losses << loss.value.value
+
+    loss.backprop
+    net.optimizer.update
+  end
+end
+```
+
+```
+...
+Epoch is: 49
+Batch id: 95
+Loss is: 0.00065050072686102
+Epoch is: 49
+Batch id: 96
+Loss is: 0.0006024037564266797
+Epoch is: 49
+Batch id: 97
+Loss is: 0.0005297538443899917
+Epoch is: 49
+Batch id: 98
+Loss is: 0.0005765025171222869
+Epoch is: 49
+Batch id: 99
+Loss is: 0.0005290653040218895
+```
+
+The Network learns this function very quickly, as XOR is one of the simplest
+distributions to hit.  Since the training data is so limited, accuracy
+can be a bit jagged, but eventually the network smooths out.
+
+### Loss over time
+
+![xorloss](xor_classifier_loss.png)

examples/basic_xor_classifier/xor.cr

@@ -0,0 +1,72 @@
+# Copyright (c) 2020 Crystal Data Contributors
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require "../../src/num"
+
+ctx = Num::Grad::Context(Tensor(Float64)).new
+
+bsz = 32
+
+x_train_bool = Tensor.random(0_u8...2_u8, [bsz * 100, 2])
+
+y_bool = x_train_bool[..., ...1] ^ x_train_bool[..., 1...]
+
+x_train = ctx.variable(x_train_bool.as_type(Float64))
+y = y_bool.as_type(Float64)
+
+net = Num::NN::Network.new(ctx) do
+  linear 2, 3
+  relu
+  linear 3, 1
+  sgd 0.7
+  sigmoid_cross_entropy_loss
+end
+
+losses = [] of Float64
+
+50.times do |epoch|
+  100.times do |batch_id|
+    offset = batch_id * 32
+    x = x_train[offset...offset + 32]
+    target = y[offset...offset + 32]
+
+    y_pred = net.forward(x)
+
+    loss = net.loss(y_pred, target)
+
+    puts "Epoch is: #{epoch}"
+    puts "Batch id: #{batch_id}"
+    puts "Loss is: #{loss.value.value}"
+    losses << loss.value.value
+
+    loss.backprop
+    net.optimizer.update
+  end
+end
+
+Num::Plot::Plot.plot do
+  scatter (0...losses.size), losses
+  x_label "Epochs"
+  y_label "Loss"
+  label "XOR Classifier Loss"
+end

examples/basic_xor_classifier/xor_classifier_loss.png

@@ -0,0 +1,15 @@
+## Simple Scatter Plot
+
+Using [PLplot](http://plplot.sourceforge.net/index.php), a fantastic library for scientific plotting, charts are extremely easy to create with Num.cr.
+
+```crystal
+x = (0...100)
+y = Tensor(Float64).rand([100])
+
+Num::Plot::Plot.plot do
+  term nil
+  scatter x, y, code: 14, color: 2
+end
+```
+
+![scatter](simple_scatter.png)

examples/simple_scatter_plot/README.md

@@ -0,0 +1,32 @@
+# Copyright (c) 2020 Crystal Data Contributors
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require "../../src/num"
+
+x = (0...100)
+y = Tensor(Float64).rand([100])
+
+Num::Plot::Plot.plot do
+  term nil
+  scatter x, y, code: 14, color: 2
+end

examples/simple_scatter_plot/scatter.cr

@@ -11,6 +11,8 @@ license: MIT
 dependencies:
   opencl:
     github: crystal-data/opencl.cr
+  alea:
+    github: nin93/alea
 
 scripts:
   postinstall: make ext

examples/simple_scatter_plot/simple_scatter.png

@@ -0,0 +1,31 @@
+require "../spec_helper"
+
+describe Tensor do
+  it "sorts a one dimensional Tensor" do
+    a = [4, 3, 2, 1].to_tensor
+    result = Num.sort(a)
+    expected = [1, 2, 3, 4].to_tensor
+    assert_array_equal(result, expected)
+  end
+
+  it "sorts a strided Tensor" do
+    a = [4, 3, 2, 1].to_tensor[{..., 2}]
+    result = Num.sort(a)
+    expected = [2, 4]
+    assert_array_equal(result, expected)
+  end
+
+  it "sorts a Tensor along an axis" do
+    a = [[3, 5, 6], [1, 1, 2], [9, 2, 3]].to_tensor
+    result = Num.sort(a, 0)
+    expected = [[1, 1, 2], [3, 2, 3], [9, 5, 6]].to_tensor
+    assert_array_equal(result, expected)
+  end
+
+  it "sorts a strided Tensor along an axis" do
+    a = [[3, 4, 5, 1], [2, 1, 3, 2], [4, 7, 6, 2]].to_tensor[..., {..., 2}]
+    result = Num.sort(a, 0)
+    expected = [[2, 3], [3, 5], [4, 6]].to_tensor
+    assert_array_equal(result, expected)
+  end
+end

shard.yml

@@ -1,5 +1,6 @@
 require "./tensor/build"
 require "./tensor/creation"
+require "./tensor/random"
 require "./tensor/linalg"
 require "./tensor/operators"
 require "./tensor/reductions"
@@ -10,11 +11,27 @@ require "./cl_tensor/cl_tensor"
 require "./cl_tensor/creation"
 require "./cl_tensor/linalg"
 
-require "./frame/frame"
-require "./frame/series"
-require "./frame/index"
-
 require "./scikit/matrices"
 require "./scikit/clustering/kmeans"
 
 require "./libs/local"
+require "./libs/nnpack"
+require "./libs/plplot"
+
+require "./grad/primitives/*"
+require "./grad/gates_arithmetic"
+require "./grad/gates_blas"
+require "./grad/variable_ops"
+
+require "./nn/primitives/*"
+require "./nn/layers/*"
+require "./nn/gates/*"
+require "./nn/optimizer"
+require "./nn/loss"
+require "./nn/network"
+
+require "./nn/datasets/*"
+
+require "./plot/internal/*"
+require "./plot/figures/*"
+require "./plot/plot"