Skip to content

A lightweight Autograd engine with a small Neural Network library written in Rust

License

Notifications You must be signed in to change notification settings

teddyrendahl/ugradrs

Folders and files

NameName
Last commit message
Last commit date

Latest commit

 

History

8 Commits
 
 
 
 
 
 
 
 
 
 
 
 
 
 

Repository files navigation

ugradrs

ugradrs

ugradrs is a tiny Autograd engine inspired by Andrej Karpathy's micrograd implemented in Rust. Intended as a personal exploration of the mechanics of neural networks. The repository allows for the creation of a DAG of scalar value operations with a small Pytorch-like API wrapper.

Example Usage

Calculate the gradient for a convoluted calculation

use ugradrs::value::Value;

let a = Value::from(-4.0);
let b = Value::from(2.0);
let mut c = a.clone() + b.clone();
let mut d = a.clone() * b.clone() + b.clone().powf(3.0.into());
c += c.clone() + 1.0;
c += c.clone() + 1.0 - a.clone();
d += d.clone() * 2.0 + (b.clone() + a.clone()).relu();
d += d.clone() * 3.0 + (b.clone() - a.clone()).relu();
let e = c.clone() - d.clone();
let f = e.clone().powf(2.0.into());
let mut g = f.clone() / 2.0;
g += Value::from(10.0) / f;

let eps = 10.0_f64.powi(-4);
assert!((g.data() - 24.7041).abs() < eps); // The outcome of the forward pass

// Perform backward propagation of gradient calculation
g.backward();

assert!((a.gradient() - 138.8338).abs() < eps); // dg/da
assert!((b.gradient() - 645.5773).abs() < eps); // dg/db

The library houses a small API for building simple Multi-Layer Perceptrons (MLPs). Both the Mlp and the SizedLayer have their input and output dimensions captured in the Rust typing system. This means that mismatches in input data and consecutive layer sizes are caught at compile time!

use ugradrs::nn::{Mlp, SizedLayer};

// By specifying the size of the hidden layer and the dimensions we ultimately want for the perceptron,
// the correct size of the input and output layers can be determined via the typing system.
let mlp: Mlp<2, 1> = Mlp::from_layer(SizedLayer::new(false)) // Adds a non-linear SizeLayer::<2, 16>
    .add_layer(SizedLayer::<16, 16>::new(false)) 
    .add_layer(SizedLayer::new(true)); // Creates a linear SizeLayer::<16, 1>

To see it in action, look at the make-moons example

$ cargo run --example make-moons --release
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
* * * * * * * * * * * * * * - - * * * * * * * * * * * * * - - 
* * * * * * * * * * * * * - - - - - * * * * * * * * * - - - - 
* * * * * * * * * * * * * - - - - - - * * * * * * * - - - - - 
* * * * * * * * * * * * - - - - - - - - * * * * * - - - - - - 
* * * * * * * * * * * - - - - - - - - - - * * * * - - - - - - 
* * * * * * * * * - - - - - - - - - - - - - * * - - - - - - - 
* * * * * * * - - - - - - - - - - - - - - - - - - - - - - - - 
* * * * * - - - - - - - - - - - - - - - - - - - - - - - - - - 
* * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 

You can also visualize created graphs with the draw_graph optional feature.

    use ugradrs::draw_dot::draw_dot;
    use ugradrs::value::Value;

    let x = Value::from(1.0);
    let y = (x * 2.0 + 1.0).relu();
    y.backward(); // Perform back-propagation to populate gradient fields
    draw_dot(y, "relu.dot").expect("Failed to create graph");

ugradrs Or draw an entire Neuron:

    use ugradrs::draw_dot::draw_dot;
    use ugradrs::nn::Neuron;
    use ugradrs::value::Value;

    let n: Neuron<2> = Neuron::new(false);
    let y = n.forward([1.0.into(), (-2.0).into()]);
    y.backward(); // Perform back-propagation to populate gradient fields
    draw_dot(y, "neuron.dot").expect("Failed to create graph");

ugradrs

To try it yourself run:

cargo run --example draw_dot --features draw_graph

Run Tests

cargo test

About

A lightweight Autograd engine with a small Neural Network library written in Rust

Resources

License

Stars

Watchers

Forks

Packages

No packages published

Languages