From 7a320fe11213cf189f2cbc2d42b87aa0b5d6e74d Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 9 Jan 2018 10:51:18 +0300 Subject: [PATCH 001/233] Add citation section --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 8231e3ff..e04e8539 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,17 @@ matvec2 = t3f.matmul(A, b_dense) # More examples For more examples (e.g. how to build neural networks or how to do Riemannian optimization) see ```examples``` folder. +# Citing +If you use T3F in your research work, we kindly ask you to cite [the paper](https://arxiv.org/abs/1801.01928) describing this library +``` +@article{novikov2018tensor, + title={Tensor Train decomposition on TensorFlow (T3F)}, + author={Novikov, Alexander and Izmailov, Pavel and Khrulkov, Valentin and Figurnov, Michael and Oseledets, Ivan}, + journal={arXiv preprint arXiv:1801.01928}, + year={2018} +} +``` + # Benchmarking Here are the results im ms of benchmarking T3F on CPU and GPU and comparing against the [TTPY library](https://github.com/oseledets/ttpy) From 8cc473de2230719e7bb31f0e8b6939e6a628578a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 16 Jan 2018 12:55:38 +0300 Subject: [PATCH 002/233] Update version number and add names to tex version --- docs/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 457abe76..b450933b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = u'0.3' +version = u'1.0' # The full version, including alpha/beta/rc tags. -release = u'0.3.0' +release = u'1.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -131,7 +131,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 't3f.tex', u't3f Documentation', - u'Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Mikhail Trofimov', 'manual'), + u'Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Mikhail Trofimov, Mikhail Trofimov, Valentin Khrulkov', 'manual'), ] From 3627f9ca9d94001e2b95badfec4a4edbd61b0d9c Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 5 Feb 2018 13:11:33 +0300 Subject: [PATCH 003/233] add quadratic form complexity --- t3f/ops.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/t3f/ops.py b/t3f/ops.py index 0ea892db..e57fd9ab 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -1028,6 +1028,15 @@ def quadratic_form(A, b, c): Raises: ValueError if the arguments are not TT-matrices or if the shapes are not consistent. + + Complexity: + O(batch_size r_A r_c r_b n d (r_b + r_A n + r_c)) + d is the number of TT-cores (A.ndims()); + r_A is the largest TT-rank of A max(A.get_tt_rank()) + n is the size of the axis dimensions e.g. + if b and c are tensors of shape (3, 3, 3), + A is a 27 x 27 matrix of tensor shape (3, 3, 3) x (3, 3, 3) + then n is 3 """ if not isinstance(A, TensorTrainBase) or not A.is_tt_matrix(): raise ValueError('The arguments should be a TT-matrix.') From f8dc5fb13a7dcddb8f689b1ab20c41037d6310d9 Mon Sep 17 00:00:00 2001 From: geom-score Date: Mon, 5 Mar 2018 16:54:12 +0300 Subject: [PATCH 004/233] should support eager mode now --- build/lib/t3f/__init__.py | 81 ++ build/lib/t3f/approximate.py | 167 +++ build/lib/t3f/approximate_test.py | 101 ++ build/lib/t3f/batch_ops.py | 218 ++++ build/lib/t3f/batch_ops_test.py | 161 +++ build/lib/t3f/decompositions.py | 599 +++++++++++ build/lib/t3f/decompositions_test.py | 176 ++++ build/lib/t3f/examples_tests.py | 43 + build/lib/t3f/initializers.py | 789 ++++++++++++++ build/lib/t3f/initializers_test.py | 176 ++++ build/lib/t3f/kronecker.py | 237 +++++ build/lib/t3f/kronecker_test.py | 182 ++++ build/lib/t3f/ops.py | 1204 ++++++++++++++++++++++ build/lib/t3f/ops_test.py | 880 ++++++++++++++++ build/lib/t3f/regularizers.py | 79 ++ build/lib/t3f/riemannian.py | 751 ++++++++++++++ build/lib/t3f/riemannian_test.py | 250 +++++ build/lib/t3f/shapes.py | 281 +++++ build/lib/t3f/tensor_train.py | 229 ++++ build/lib/t3f/tensor_train_base.py | 189 ++++ build/lib/t3f/tensor_train_batch.py | 367 +++++++ build/lib/t3f/tensor_train_batch_test.py | 77 ++ build/lib/t3f/tensor_train_test.py | 138 +++ build/lib/t3f/utils.py | 39 + build/lib/t3f/utils_test.py | 26 + build/lib/t3f/variables.py | 141 +++ build/lib/t3f/variables_test.py | 82 ++ t3f/variables.py | 14 +- 28 files changed, 7676 insertions(+), 1 deletion(-) create mode 100644 build/lib/t3f/__init__.py create mode 100644 build/lib/t3f/approximate.py create mode 100644 build/lib/t3f/approximate_test.py create mode 100644 build/lib/t3f/batch_ops.py create mode 100644 build/lib/t3f/batch_ops_test.py create mode 100644 build/lib/t3f/decompositions.py create mode 100644 build/lib/t3f/decompositions_test.py create mode 100644 build/lib/t3f/examples_tests.py create mode 100644 build/lib/t3f/initializers.py create mode 100644 build/lib/t3f/initializers_test.py create mode 100644 build/lib/t3f/kronecker.py create mode 100644 build/lib/t3f/kronecker_test.py create mode 100644 build/lib/t3f/ops.py create mode 100644 build/lib/t3f/ops_test.py create mode 100644 build/lib/t3f/regularizers.py create mode 100644 build/lib/t3f/riemannian.py create mode 100644 build/lib/t3f/riemannian_test.py create mode 100644 build/lib/t3f/shapes.py create mode 100644 build/lib/t3f/tensor_train.py create mode 100644 build/lib/t3f/tensor_train_base.py create mode 100644 build/lib/t3f/tensor_train_batch.py create mode 100644 build/lib/t3f/tensor_train_batch_test.py create mode 100644 build/lib/t3f/tensor_train_test.py create mode 100644 build/lib/t3f/utils.py create mode 100644 build/lib/t3f/utils_test.py create mode 100644 build/lib/t3f/variables.py create mode 100644 build/lib/t3f/variables_test.py diff --git a/build/lib/t3f/__init__.py b/build/lib/t3f/__init__.py new file mode 100644 index 00000000..60abb533 --- /dev/null +++ b/build/lib/t3f/__init__.py @@ -0,0 +1,81 @@ +from t3f.tensor_train_base import TensorTrainBase +from t3f.tensor_train import TensorTrain +from t3f.tensor_train_batch import TensorTrainBatch + +from t3f.variables import assign +from t3f.variables import get_variable + +from t3f.ops import add +from t3f.ops import cast +from t3f.ops import flat_inner +from t3f.ops import frobenius_norm +from t3f.ops import frobenius_norm_squared +from t3f.ops import full +from t3f.ops import matmul +from t3f.ops import multiply +from t3f.ops import quadratic_form +from t3f.ops import transpose +from t3f.ops import gather_nd +from t3f.ops import renormalize_tt_cores + +from t3f.batch_ops import concat_along_batch_dim +from t3f.batch_ops import gram_matrix +from t3f.batch_ops import multiply_along_batch_dim +from t3f.batch_ops import pairwise_flat_inner + +from t3f.initializers import matrix_with_random_cores +from t3f.initializers import matrix_batch_with_random_cores +from t3f.initializers import tensor_with_random_cores +from t3f.initializers import tensor_batch_with_random_cores +from t3f.initializers import random_tensor +from t3f.initializers import random_tensor_batch +from t3f.initializers import random_matrix +from t3f.initializers import random_matrix_batch +from t3f.initializers import tensor_ones +from t3f.initializers import tensor_zeros +from t3f.initializers import matrix_ones +from t3f.initializers import matrix_zeros +from t3f.initializers import eye +from t3f.initializers import ones_like +from t3f.initializers import zeros_like +from t3f.initializers import glorot_initializer +from t3f.initializers import he_initializer +from t3f.initializers import lecun_initializer + +from t3f.regularizers import cores_regularizer +from t3f.regularizers import l2_regularizer + +from t3f.riemannian import add_n_projected +from t3f.riemannian import pairwise_flat_inner_projected +from t3f.riemannian import project +from t3f.riemannian import project_matmul +from t3f.riemannian import project_sum + +from t3f.shapes import batch_size +from t3f.shapes import clean_raw_shape +from t3f.shapes import expand_batch_dim +from t3f.shapes import is_batch_broadcasting_possible +from t3f.shapes import lazy_batch_size +from t3f.shapes import lazy_raw_shape +from t3f.shapes import lazy_shape +from t3f.shapes import lazy_tt_ranks +from t3f.shapes import raw_shape +from t3f.shapes import shape +from t3f.shapes import squeeze_batch_dim +from t3f.shapes import tt_ranks + +from t3f.decompositions import orthogonalize_tt_cores +from t3f.decompositions import round +from t3f.decompositions import to_tt_matrix +from t3f.decompositions import to_tt_tensor + +import t3f.approximate as approximate +import t3f.kronecker as kronecker +import t3f.utils as utils + +_directly_imported = ['tensor_train_base', 'tensor_train', 'tensor_train_batch', + 'variables', 'ops', 'batch_ops', 'initializers', + 'regularizers', 'riemannian', 'shapes', 'decompositions'] + +__all__ = [s for s in dir() if + s not in _directly_imported and not s.startswith('_')] diff --git a/build/lib/t3f/approximate.py b/build/lib/t3f/approximate.py new file mode 100644 index 00000000..4cb8f495 --- /dev/null +++ b/build/lib/t3f/approximate.py @@ -0,0 +1,167 @@ +import numpy as np +import tensorflow as tf +from t3f.tensor_train_batch import TensorTrainBatch +from t3f import decompositions +from t3f import batch_ops + + +def add_n(tt_objects, max_tt_rank): + """Adds a bunch of TT-object and round after each summation. + + This version implements a slow-to-compile but fast-to-execute (at least on + a GPU) version: summing in a binary tree order. + I.e. it uses the following idea: + round(a + b + c + d) ~= round(round(a + b) + round(c + d)) + and so is able to compute the answer in log(N) parallel adds/rounds. + + Args: + tt_objects: a list of `TensorTrainBase` objects. + max_tt_rank: a number, TT-rank for each individual rounding. + + Returns: + Object of the same type as each input. + + See Also: + t3f.approximate.reduce_sum_batch + """ + prev_level = tt_objects + while len(prev_level) > 1: + next_level = [] + for i in range(0, len(prev_level), 2): + curr = prev_level[i] + if i + 1 < len(prev_level): + curr = decompositions.round(curr + prev_level[i + 1], max_tt_rank) + next_level.append(curr) + prev_level = next_level + return prev_level[0] + + +def reduce_sum_batch(tt_batch, max_tt_rank, coef=None): + """Sum of all TT-objects in the batch with rounding after each summation. + + This version implements a slow-to-compile but fast-to-execute (at least on + a GPU) version: summing in a binary tree order. + I.e. it uses the following idea: + round(a + b + c + d) ~= round(round(a + b) + round(c + d)) + and so is able to compute the answer in log(batch_size) parallel adds/rounds. + + Args: + tt_batch: `TensorTrainBatch` object. + max_tt_rank: a number, TT-rank for each individual rounding. + coef: tf.Tensor, its shape is either batch_size, or batch_size x N. + If coef is a vecotor of size batch_size, the result will + be (approximate) weighted sum. + If coef is a matrix of shape batch_size x N, the result will be + a `TensorTrainBatch` res containing N TT-object such that + res[j] ~= sum_i tt_batch[i] coef[i, j] + + Returns: + If coefficients are absent or is a vector of numbers, returns + a `TensorTrain` object representing (approximate) element-wise sum of all + the objects in the batch, weighted if coef is provided. + If coefficients is a matrix, returns `TensorTrainBatch`. + + See Also: + t3f.approximate.add_n + """ + ndims = tt_batch.ndims() + left_tt_rank_dim = tt_batch.left_tt_rank_dim + right_tt_rank_dim = tt_batch.right_tt_rank_dim + shape = tt_batch.get_raw_shape() + dtype = tt_batch.dtype + + is_batch_output = False + if coef is not None: + coef = tf.convert_to_tensor(coef) + if len(coef.get_shape()) == 1: + tt_batch = batch_ops.multiply_along_batch_dim(tt_batch, coef) + elif len(coef.get_shape()) == 2: + is_batch_output = True + output_size = coef.get_shape().as_list()[1] + # Coef is of size batch_size x N, need to duplicate the batch + # dimension xN. + if coef.shape[0] != tt_batch.batch_size: + raise ValueError('If coef is a matrix, it should be of shape ' + 'batch_size x N, got %d x %d instead ' + '(batch size is %d).' % (coef.shape[0], coef.shape[1], + tt_batch.batch_size)) + tt_batch_cores = [] + for core_idx in range(ndims): + curr_core = tt_batch.tt_cores[core_idx] + curr_shape = curr_core.get_shape().as_list() + new_shape = np.insert(curr_shape, 1, 1) + tiling = np.ones(len(new_shape)) + tiling[1] = output_size + curr_core = tf.tile(tf.reshape(curr_core, new_shape), tiling) + if core_idx == 0: + # Multiply the first TT-core by the provided coefficients. + # TODO: add t3f.utils.expands_dims_like(coef, curr_core) + shaped_coef = coef + for _ in range(len(curr_core.get_shape()) - len(coef.shape)): + shaped_coef = tf.expand_dims(shaped_coef, -1) + curr_core = curr_core * shaped_coef + # Merge the first two dimensions back into one. + raveled_shape = np.array(curr_shape).copy() + raveled_shape[0] *= output_size + curr_core = tf.reshape(curr_core, raveled_shape) + tt_batch_cores.append(curr_core) + tt_batch = TensorTrainBatch(tt_batch_cores, shape, + tt_batch.get_tt_ranks()) + + else: + raise ValueError('Coef cannot be more than 2-d.') + + if not is_batch_output: + output_size = 1 + + prev_level = tt_batch + while prev_level.batch_size > output_size: + current_level_cores = [] + for core_idx in range(ndims): + curr_orig_core = prev_level.tt_cores[core_idx] + if is_batch_output: + # Split the first dimension into batch_size x N + unraveled_shape = curr_orig_core.get_shape().as_list() + unraveled_shape = np.array(unraveled_shape).copy() + unraveled_shape[0] /= output_size + unraveled_shape = np.insert(unraveled_shape, 1, output_size) + curr_orig_core = tf.reshape(curr_orig_core, unraveled_shape) + + a_core = curr_orig_core[::2] + b_core = curr_orig_core[1::2] + + if a_core.get_shape()[0] > b_core.get_shape()[0]: + # Odd number of elements in the batch, will have to add dummy + # TT-object with the tt-cores filled with zeros. + zeros_shape = b_core.get_shape().as_list() + zeros_shape[0] = 1 + zeros = tf.zeros(zeros_shape, dtype) + b_core = tf.concat((b_core, zeros), axis=0) + + if is_batch_output: + # Merge the first two dimensions back into one. + a_core_shape = a_core.get_shape().as_list() + a_core_shape[0] = a_core_shape[0] * a_core_shape[1] + a_core_shape = np.delete(a_core_shape, 1) + a_core = tf.reshape(a_core, a_core_shape) + b_core_shape = b_core.get_shape().as_list() + b_core_shape[0] = b_core_shape[0] * b_core_shape[1] + b_core_shape = np.delete(b_core_shape, 1) + b_core = tf.reshape(b_core, b_core_shape) + + if core_idx == 0: + curr_sum_core = tf.concat((a_core, b_core), axis=right_tt_rank_dim) + elif core_idx == ndims - 1: + curr_sum_core = tf.concat((a_core, b_core), axis=left_tt_rank_dim) + else: + zeros = tf.zeros(b_core.get_shape(), dtype) + upper = tf.concat((a_core, zeros), axis=right_tt_rank_dim) + lower = tf.concat((zeros, b_core), axis=right_tt_rank_dim) + curr_sum_core = tf.concat((upper, lower), axis=left_tt_rank_dim) + current_level_cores.append(curr_sum_core) + current_level = TensorTrainBatch(current_level_cores, shape) + prev_level = decompositions.round(current_level, max_tt_rank) + if is_batch_output: + return prev_level + else: + return prev_level[0] diff --git a/build/lib/t3f/approximate_test.py b/build/lib/t3f/approximate_test.py new file mode 100644 index 00000000..52a99320 --- /dev/null +++ b/build/lib/t3f/approximate_test.py @@ -0,0 +1,101 @@ +import numpy as np +import tensorflow as tf + +from t3f import ops +from t3f import approximate +from t3f import initializers + + +class ApproximateTest(tf.test.TestCase): + + def testAddN(self): + # Sum a bunch of TT-matrices. + tt_a = initializers.random_matrix(((2, 1, 4), (2, 2, 2)), tt_rank=2) + tt_b = initializers.random_matrix(((2, 1, 4), (2, 2, 2)), + tt_rank=[1, 2, 4, 1]) + + def desired(tt_objects): + res = tt_objects[0] + for tt in tt_objects[1:]: + res += tt + return res + + with self.test_session() as sess: + res_actual = ops.full(approximate.add_n([tt_a, tt_b], 6)) + res_desired = ops.full(desired([tt_a, tt_b])) + res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) + self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) + + res_actual = ops.full(approximate.add_n([tt_a, tt_b, tt_a], 8)) + res_desired = ops.full(desired([tt_a, tt_b, tt_a])) + res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) + self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) + + res_actual = ops.full(approximate.add_n([tt_a, tt_b, tt_a, tt_a, tt_a], 12)) + res_desired = ops.full(desired([tt_a, tt_b, tt_a, tt_a, tt_a])) + res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) + self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) + + def testReduceSumBatch(self): + # Sum a batch of TT-tensors. + + def desired(tt_batch): + res = tt_batch[0] + for i in range(1, tt_batch.batch_size): + res += tt_batch[i] + return res + for batch_size in [2, 3, 4, 5]: + with self.test_session() as sess: + tt_batch = initializers.random_tensor_batch((4, 3, 5), + tt_rank=2, + batch_size=batch_size) + res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 10)) + res_desired = ops.full(desired(tt_batch)) + res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) + self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) + + def testReduceSumBatchWeighted(self): + # Weighted sum of a batch of TT-tensors. + + def desired(tt_batch, coef): + res = coef[0] * tt_batch[0] + for i in range(1, tt_batch.batch_size): + res += coef[i] * tt_batch[i] + return res + with self.test_session() as sess: + tt_batch = initializers.random_tensor_batch((4, 3, 5), + tt_rank=3, + batch_size=3) + res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 9, + [1.2, -0.2, 1])) + res_desired = ops.full(desired(tt_batch, [1.2, -0.2, 1])) + res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) + self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) + + def testReduceSumBatchMultipleWeighted(self): + # Multiple weighted sums of a batch of TT-tensors. + + def desired(tt_batch, coef): + res = coef[0] * tt_batch[0] + for i in range(1, tt_batch.batch_size): + res += coef[i] * tt_batch[i] + return res + with self.test_session() as sess: + tt_batch = initializers.random_tensor_batch((4, 3, 5), + tt_rank=2, + batch_size=3) + coef = [[1., 0.1], + [0.9, -0.2], + [0.3, 0.3]] + coef = np.array(coef).astype(np.float32) + res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 6, + coef)) + res_desired_1 = ops.full(desired(tt_batch, coef[:, 0])) + res_desired_2 = ops.full(desired(tt_batch, coef[:, 1])) + res_desired = tf.stack((res_desired_1, res_desired_2)) + res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) + self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) + + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/batch_ops.py b/build/lib/t3f/batch_ops.py new file mode 100644 index 00000000..bd89bb36 --- /dev/null +++ b/build/lib/t3f/batch_ops.py @@ -0,0 +1,218 @@ +import tensorflow as tf + +from t3f.tensor_train_base import TensorTrainBase +from t3f.tensor_train_batch import TensorTrainBatch +from t3f import ops + + +def concat_along_batch_dim(tt_list): + """Concat all TensorTrainBatch objects along batch dimension. + + Args: + tt_list: a list of TensorTrainBatch objects. + + Returns: + TensorTrainBatch + """ + ndims = tt_list[0].ndims() + + if isinstance(tt_list, TensorTrainBase): + # Not a list but just one element, nothing to concat. + return tt_list + + for batch_idx in range(len(tt_list)): + if not isinstance(tt_list[batch_idx], TensorTrainBatch): + raise ValueError('All objects in the list should be TTBatch objects, got ' + '%s' % tt_list[batch_idx]) + for batch_idx in range(1, len(tt_list)): + if tt_list[batch_idx].get_raw_shape() != tt_list[0].get_raw_shape(): + raise ValueError('Shapes of all TT-batch objects should coincide, got %s ' + 'and %s' % (tt_list[0].get_raw_shape(), + tt_list[batch_idx].get_raw_shape())) + if tt_list[batch_idx].get_tt_ranks() != tt_list[0].get_tt_ranks(): + raise ValueError('TT-ranks of all TT-batch objects should coincide, got ' + '%s and %s' % (tt_list[0].get_tt_ranks(), + tt_list[batch_idx].get_tt_ranks())) + + res_cores = [] + for core_idx in range(ndims): + curr_core = tf.concat([tt.tt_cores[core_idx] for tt in tt_list], axis=0) + res_cores.append(curr_core) + + try: + batch_size = sum([tt.batch_size for tt in tt_list]) + except TypeError: + # The batch sizes are not defined and you can't sum Nones. + batch_size = None + + return TensorTrainBatch(res_cores, tt_list[0].get_raw_shape(), + tt_list[0].get_tt_ranks(), batch_size) + + +def multiply_along_batch_dim(batch_tt, weights): + """Multiply each TensorTrain in a batch by a number. + + Args: + batch_tt: TensorTrainBatch object, TT-matrices or TT-tensors. + weights: 1-D tf.Tensor (or something convertible to it like np.array) of size + tt.batch_size with weights. + + Returns: + TensorTrainBatch + """ + weights = tf.convert_to_tensor(weights) + tt_cores = list(batch_tt.tt_cores) + if batch_tt.is_tt_matrix(): + weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis, tf.newaxis] + else: + weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis] + tt_cores[0] = weights * tt_cores[0] + out_shape = batch_tt.get_raw_shape() + out_ranks = batch_tt.get_tt_ranks() + out_batch_size = batch_tt.batch_size + return TensorTrainBatch(tt_cores, out_shape, out_ranks, out_batch_size) + + +def gram_matrix(tt_vectors, matrix=None): + """Computes Gramian matrix of a batch of TT-vectors. + + If matrix is None, computes + res[i, j] = t3f.flat_inner(tt_vectors[i], tt_vectors[j]). + If matrix is present, computes + res[i, j] = t3f.flat_inner(tt_vectors[i], t3f.matmul(matrix, tt_vectors[j])) + or more shortly + res[i, j] = tt_vectors[i]^T * matrix * tt_vectors[j] + but is more efficient. + + Args: + tt_vectors: TensorTrainBatch. + matrix: None, or TensorTrain matrix. + + Returns: + tf.tensor with the Gram matrix. + + Complexity: + If the matrix is not present, the complexity is O(batch_size^2 d r^3 n) + where d is the number of + TT-cores (tt_vectors.ndims()), r is the largest TT-rank + max(tt_vectors.get_tt_rank()) + and n is the size of the axis dimension, e.g. + for a tensor of size 4 x 4 x 4, n is 4; + for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 + If the matrix of TT-rank R is present, the complexity is + O(batch_size^2 d R r^2 n (r + nR)) + where the matrix is of raw-shape (n, n, ..., n) x (n, n, ..., n); + r is the TT-rank of vectors tt_vectors; + R is the TT-rank of the matrix. + """ + return pairwise_flat_inner(tt_vectors, tt_vectors, matrix) + + +def pairwise_flat_inner(tt_1, tt_2, matrix=None): + """Computes all scalar products between two batches of TT-objects. + + If matrix is None, computes + res[i, j] = t3f.flat_inner(tt_1[i], tt_2[j]). + + If matrix is present, computes + res[i, j] = t3f.flat_inner(tt_1[i], t3f.matmul(matrix, tt_2[j])) + or more shortly + res[i, j] = tt_1[i]^T * matrix * tt_2[j] + but is more efficient. + + Args: + tt_1: TensorTrainBatch. + tt_2: TensorTrainBatch. + matrix: None, or TensorTrain matrix. + + Returns: + tf.tensor with the matrix of pairwise scalar products (flat inners). + + Complexity: + If the matrix is not present, the complexity is O(batch_size^2 d r^3 n) + where d is the number of + TT-cores (tt_vectors.ndims()), r is the largest TT-rank + max(tt_vectors.get_tt_rank()) + and n is the size of the axis dimension, e.g. + for a tensor of size 4 x 4 x 4, n is 4; + for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 + A more precise complexity is + O(batch_size^2 d r1 r2 n max(r1, r2)) + where r1 is the largest TT-rank of tt_a + and r2 is the largest TT-rank of tt_b. + If the matrix is present, the complexity is + O(batch_size^2 d R r1 r2 (n r1 + n m R + m r2)) + where + the matrix is of raw-shape (n, n, ..., n) x (m, m, ..., m) and TT-rank R; + tt_1 is of shape (n, n, ..., n) and is of the TT-rank r1; + tt_2 is of shape (m, m, ..., m) and is of the TT-rank r2; + """ + ndims = tt_1.ndims() + if matrix is None: + curr_core_1 = tt_1.tt_cores[0] + curr_core_2 = tt_2.tt_cores[0] + mode_string = 'ij' if tt_1.is_tt_matrix() else 'i' + einsum_str = 'pa{0}b,qc{0}d->pqbd'.format(mode_string) + res = tf.einsum(einsum_str, curr_core_1, curr_core_2) + for core_idx in range(1, ndims): + curr_core_1 = tt_1.tt_cores[core_idx] + curr_core_2 = tt_2.tt_cores[core_idx] + einsum_str = 'pqac,pa{0}b,qc{0}d->pqbd'.format(mode_string) + res = tf.einsum(einsum_str, res, curr_core_1, curr_core_2) + else: + # res[i, j] = tt_1[i] ^ T * matrix * tt_2[j] + if not tt_1.is_tt_matrix() or not tt_2.is_tt_matrix() or not matrix.is_tt_matrix(): + raise ValueError('When passing three arguments to pairwise_flat_inner, ' + 'the first 2 of them should be TT-vecors and the last ' + 'should be a TT-matrix. Got %s, %s, and %s instead.' % + (tt_1, tt_2, matrix)) + matrix_shape = matrix.get_raw_shape() + if not tt_1.get_raw_shape()[0].is_compatible_with(matrix_shape[0]): + raise ValueError('The shape of the first argument should be compatible ' + 'with the shape of the TT-matrix, that is it should be ' + 'possible to do the following matmul: ' + 'transpose(tt_1) * matrix. Got the first argument ' + '"%s" and matrix "%s"' % (tt_1, matrix)) + if not tt_2.get_raw_shape()[0].is_compatible_with(matrix_shape[1]): + raise ValueError('The shape of the second argument should be compatible ' + 'with the shape of the TT-matrix, that is it should be ' + 'possible to do the following matmul: ' + 'matrix * tt_2. Got the second argument ' + '"%s" and matrix "%s"' % (tt_2, matrix)) + + vectors_1_shape = tt_1.get_shape() + if vectors_1_shape[2] == 1 and vectors_1_shape[1] != 1: + # TODO: not very efficient, better to use different order in einsum. + tt_1 = ops.transpose(tt_1) + vectors_1_shape = tt_1.get_shape() + vectors_2_shape = tt_2.get_shape() + if vectors_2_shape[2] == 1 and vectors_2_shape[1] != 1: + # TODO: not very efficient, better to use different order in einsum. + tt_2 = ops.transpose(tt_2) + vectors_2_shape = tt_2.get_shape() + if vectors_1_shape[1] != 1: + # TODO: do something so that in case the shape is undefined on compilation + # it still works. + raise ValueError('The tt_vectors_1 argument should be vectors (not ' + 'matrices) with shape defined on compilation.') + if vectors_2_shape[1] != 1: + # TODO: do something so that in case the shape is undefined on compilation + # it still works. + raise ValueError('The tt_vectors_2 argument should be vectors (not ' + 'matrices) with shape defined on compilation.') + curr_core_1 = tt_1.tt_cores[0] + curr_core_2 = tt_2.tt_cores[0] + curr_matrix_core = matrix.tt_cores[0] + # We enumerate the dummy dimension (that takes 1 value) with `k`. + res = tf.einsum('pakib,cijd,qekjf->pqbdf', curr_core_1, curr_matrix_core, + curr_core_2) + for core_idx in range(1, ndims): + curr_core_1 = tt_1.tt_cores[core_idx] + curr_core_2 = tt_2.tt_cores[core_idx] + curr_matrix_core = matrix.tt_cores[core_idx] + res = tf.einsum('pqace,pakib,cijd,qekjf->pqbdf', res, curr_core_1, + curr_matrix_core, curr_core_2) + + # Squeeze to make the result of size batch_size x batch_size instead of + # batch_size x batch_size x 1 x 1. + return tf.squeeze(res) diff --git a/build/lib/t3f/batch_ops_test.py b/build/lib/t3f/batch_ops_test.py new file mode 100644 index 00000000..4c8cfd17 --- /dev/null +++ b/build/lib/t3f/batch_ops_test.py @@ -0,0 +1,161 @@ +import numpy as np +import tensorflow as tf + +from t3f import ops +from t3f import batch_ops +from t3f import initializers + + +class BatchOpsTest(tf.test.TestCase): + + def testConcatMatrix(self): + # Test concating TTMatrix batches along batch dimension. + first = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=1) + second = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=4) + third = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3) + first_res = batch_ops.concat_along_batch_dim((first)) + first_res = ops.full(first_res) + first_second_res = batch_ops.concat_along_batch_dim((first, second)) + first_second_res = ops.full(first_second_res) + first_second_third_res = batch_ops.concat_along_batch_dim((first, second, + third)) + first_second_third_res = ops.full(first_second_third_res) + + first_full = ops.full(first) + second_full = ops.full(second) + third_full = ops.full(third) + first_desired = first_full + first_second_desired = tf.concat((first_full, second_full), axis=0) + first_second_third_desired = tf.concat((first_full, second_full, third_full), + axis=0) + with self.test_session() as sess: + res = sess.run((first_res, first_second_res, first_second_third_res, + first_desired, first_second_desired, + first_second_third_desired)) + first_res_val = res[0] + first_second_res_val = res[1] + first_second_third_res_val = res[2] + first_desired_val = res[3] + first_second_desired_val = res[4] + first_second_third_desired_val = res[5] + self.assertAllClose(first_res_val, first_desired_val) + self.assertAllClose(first_second_res_val, first_second_desired_val) + self.assertAllClose(first_second_third_res_val, first_second_third_desired_val) + + def testConcatTensorPlaceholders(self): + # Test concating TTTensors of unknown batch sizes along batch dimension. + number_of_objects = tf.placeholder(tf.int32) + all = initializers.random_tensor_batch((2, 3), batch_size=5) + actual = batch_ops.concat_along_batch_dim((all[:number_of_objects], + all[number_of_objects:])) + with self.test_session() as sess: + desired_val, actual_val = sess.run((ops.full(all), ops.full(actual)), + feed_dict={number_of_objects: 2}) + self.assertAllClose(desired_val, actual_val) + + def testConcatMatrixPlaceholders(self): + # Test concating TTMatrices of unknown batch sizes along batch dimension. + number_of_objects = tf.placeholder(tf.int32) + all = initializers.random_matrix_batch(((2, 3), (2, 3)), batch_size=5) + actual = batch_ops.concat_along_batch_dim((all[:number_of_objects], + all[number_of_objects:])) + with self.test_session() as sess: + desired_val, actual_val = sess.run((ops.full(all), ops.full(actual)), + feed_dict={number_of_objects: 2}) + self.assertAllClose(desired_val, actual_val) + + def testBatchMultiply(self): + # Test multiplying batch of TTMatrices by individual numbers. + tt = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3) + weights = [0.1, 0, -10] + actual = batch_ops.multiply_along_batch_dim(tt, weights) + individual_desired = [weights[i] * tt[i:i+1] for i in range(3)] + desired = batch_ops.concat_along_batch_dim(individual_desired) + with self.test_session() as sess: + desired_val, acutual_val = sess.run((ops.full(desired), ops.full(actual))) + self.assertAllClose(desired_val, acutual_val) + + def testGramMatrix(self): + # Test Gram Matrix of a batch of TT vectors. + tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=5) + res_actual = batch_ops.gram_matrix(tt_vectors) + full_vectors = tf.reshape(ops.full(tt_vectors), (5, 6)) + res_desired = tf.matmul(full_vectors, tf.transpose(full_vectors)) + res_desired = tf.squeeze(res_desired) + with self.test_session() as sess: + res_actual_val, res_desired_val = sess.run((res_actual, res_desired)) + self.assertAllClose(res_desired_val, res_actual_val) + + def testGramMatrixWithMatrix(self): + # Test Gram Matrix of a batch of TT vectors with providing a matrix, so we + # should compute + # res[i, j] = tt_vectors[i] ^ T * matrix * tt_vectors[j] + tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=4) + matrix = initializers.random_matrix(((2, 3), (2, 3))) + res_actual = batch_ops.gram_matrix(tt_vectors, matrix) + full_vectors = tf.reshape(ops.full(tt_vectors), (4, 6)) + with self.test_session() as sess: + res = sess.run((res_actual, full_vectors, ops.full(matrix))) + res_actual_val, vectors_val, matrix_val = res + res_desired_val = np.zeros((4, 4)) + for i in range(4): + for j in range(4): + curr_val = np.dot(vectors_val[i], matrix_val) + curr_val = np.dot(curr_val, vectors_val[j]) + res_desired_val[i, j] = curr_val + self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) + + def testPairwiseFlatInnerTensor(self): + # Test pairwise_flat_inner of a batch of TT tensors. + tt_tensors_1 = initializers.random_tensor_batch((2, 3, 2), batch_size=5) + tt_tensors_2 = initializers.random_tensor_batch((2, 3, 2), batch_size=5) + res_actual = batch_ops.pairwise_flat_inner(tt_tensors_1, tt_tensors_2) + full_tensors_1 = tf.reshape(ops.full(tt_tensors_1), (5, 12)) + full_tensors_2 = tf.reshape(ops.full(tt_tensors_2), (5, 12)) + res_desired = tf.matmul(full_tensors_1, tf.transpose(full_tensors_2)) + res_desired = tf.squeeze(res_desired) + with self.test_session() as sess: + res_actual_val, res_desired_val = sess.run((res_actual, res_desired)) + self.assertAllClose(res_desired_val, res_actual_val) + + def testPairwiseFlatInnerMatrix(self): + # Test pairwise_flat_inner of a batch of TT matrices. + tt_vectors_1 = initializers.random_matrix_batch(((2, 3), (2, 3)), + batch_size=5) + tt_vectors_2 = initializers.random_matrix_batch(((2, 3), (2, 3)), + batch_size=5) + res_actual = batch_ops.pairwise_flat_inner(tt_vectors_1, tt_vectors_2) + full_vectors_1 = tf.reshape(ops.full(tt_vectors_1), (5, 36)) + full_vectors_2 = tf.reshape(ops.full(tt_vectors_2), (5, 36)) + res_desired = tf.matmul(full_vectors_1, tf.transpose(full_vectors_2)) + res_desired = tf.squeeze(res_desired) + with self.test_session() as sess: + res_actual_val, res_desired_val = sess.run((res_actual, res_desired)) + self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) + + def testPairwiseFlatInnerVectorsWithMatrix(self): + # Test pairwise_flat_inner of a batch of TT vectors with providing a matrix, + # so we should compute + # res[i, j] = tt_vectors[i] ^ T * matrix * tt_vectors[j] + tt_vectors_1 = initializers.random_matrix_batch(((2, 3), None), batch_size=2) + tt_vectors_2 = initializers.random_matrix_batch(((2, 3), None), batch_size=3) + matrix = initializers.random_matrix(((2, 3), (2, 3))) + res_actual = batch_ops.pairwise_flat_inner(tt_vectors_1, tt_vectors_2, + matrix) + full_vectors_1 = tf.reshape(ops.full(tt_vectors_1), (2, 6)) + full_vectors_2 = tf.reshape(ops.full(tt_vectors_2), (3, 6)) + with self.test_session() as sess: + res = sess.run((res_actual, full_vectors_1, full_vectors_2, + ops.full(matrix))) + res_actual_val, vectors_1_val, vectors_2_val, matrix_val = res + res_desired_val = np.zeros((2, 3)) + for i in range(2): + for j in range(3): + curr_val = np.dot(vectors_1_val[i], matrix_val) + curr_val = np.dot(curr_val, vectors_2_val[j]) + res_desired_val[i, j] = curr_val + self.assertAllClose(res_desired_val, res_actual_val) + +if __name__ == "__main__": + tf.test.main() + diff --git a/build/lib/t3f/decompositions.py b/build/lib/t3f/decompositions.py new file mode 100644 index 00000000..b16ee7f0 --- /dev/null +++ b/build/lib/t3f/decompositions.py @@ -0,0 +1,599 @@ +import numpy as np +import tensorflow as tf + +from t3f.tensor_train import TensorTrain +from t3f.tensor_train_batch import TensorTrainBatch +from t3f import shapes + + +def to_tt_matrix(mat, shape, max_tt_rank=10, epsilon=None): + """Converts a given matrix or vector to a TT-matrix. + + The matrix dimensions should factorize into d numbers. + If e.g. the dimensions are prime numbers, it's usually better to + pad the matrix with zeros until the dimensions factorize into + (ideally) 3-8 numbers. + + Args: + mat: two dimensional tf.Tensor (a matrix). + shape: two dimensional array (np.array or list of lists) + Represents the tensor shape of the matrix. + E.g. for a (a1 * a2 * a3) x (b1 * b2 * b3) matrix `shape` should be + ((a1, a2, a3), (b1, b2, b3)) + `shape[0]`` and `shape[1]`` should have the same length. + For vectors you may use ((a1, a2, a3), (1, 1, 1)) or, equivalently, + ((a1, a2, a3), None) + max_tt_rank: a number or a list of numbers + If a number, than defines the maximal TT-rank of the result. + If a list of numbers, than `max_tt_rank` length should be d+1 + (where d is the length of `shape[0]`) and `max_tt_rank[i]` defines + the maximal (i+1)-th TT-rank of the result. + The following two versions are equivalent + `max_tt_rank = r` + and + `max_tt_rank = r * np.ones(d-1)` + epsilon: a floating point number or None + If the TT-ranks are not restricted (`max_tt_rank=np.inf`), then + the result would be guarantied to be `epsilon` close to `mat` + in terms of relative Frobenius error: + ||res - mat||_F / ||mat||_F <= epsilon + If the TT-ranks are restricted, providing a loose `epsilon` may reduce + the TT-ranks of the result. + E.g. + to_tt_matrix(mat, shape, max_tt_rank=100, epsilon=0.9) + will probably return you a TT-matrix with TT-ranks close to 1, not 100. + Note that providing a nontrivial (= not equal to None) `epsilon` will make + the TT-ranks of the result undefined on the compilation stage + (e.g. res.get_tt_ranks() will return None, but t3f.tt_ranks(res).eval() + will work). + + Returns: + `TensorTrain` object containing a TT-matrix. + + Raises: + ValueError if max_tt_rank is less than 0, if max_tt_rank is not a number and + not a vector of length d + 1 where d is the number of dimensions (rank) of + the input tensor, if epsilon is less than 0. + """ + mat = tf.convert_to_tensor(mat) + # In case the shape is immutable. + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1])).astype(int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0])).astype(int) + + shape = np.array(shape) + tens = tf.reshape(mat, shape.flatten()) + d = len(shape[0]) + # transpose_idx = 0, d, 1, d+1 ... + transpose_idx = np.arange(2 * d).reshape(2, d).T.flatten() + transpose_idx = transpose_idx.astype(int) + tens = tf.transpose(tens, transpose_idx) + new_shape = np.prod(shape, axis=0) + tens = tf.reshape(tens, new_shape) + tt_tens = to_tt_tensor(tens, max_tt_rank, epsilon) + tt_cores = [] + static_tt_ranks = tt_tens.get_tt_ranks() + dynamic_tt_ranks = shapes.tt_ranks(tt_tens) + for core_idx in range(d): + curr_core = tt_tens.tt_cores[core_idx] + curr_rank = static_tt_ranks[core_idx].value + if curr_rank is None: + curr_rank = dynamic_tt_ranks[core_idx] + next_rank = static_tt_ranks[core_idx + 1].value + if next_rank is None: + next_rank = dynamic_tt_ranks[core_idx + 1] + curr_core_new_shape = (curr_rank, shape[0, core_idx], + shape[1, core_idx], next_rank) + curr_core = tf.reshape(curr_core, curr_core_new_shape) + tt_cores.append(curr_core) + return TensorTrain(tt_cores, shape, tt_tens.get_tt_ranks()) + + +# TODO: implement epsilon. +def to_tt_tensor(tens, max_tt_rank=10, epsilon=None): + """Converts a given tf.Tensor to a TT-tensor of the same shape. + + Args: + tens: tf.Tensor + max_tt_rank: a number or a list of numbers + If a number, than defines the maximal TT-rank of the result. + If a list of numbers, than `max_tt_rank` length should be d+1 + (where d is the rank of `tens`) and `max_tt_rank[i]` defines + the maximal (i+1)-th TT-rank of the result. + The following two versions are equivalent + `max_tt_rank = r` + and + `max_tt_rank = r * np.ones(d-1)` + epsilon: a floating point number or None + If the TT-ranks are not restricted (`max_tt_rank=np.inf`), then + the result would be guarantied to be `epsilon` close to `tens` + in terms of relative Frobenius error: + ||res - tens||_F / ||tens||_F <= epsilon + If the TT-ranks are restricted, providing a loose `epsilon` may + reduce the TT-ranks of the result. + E.g. + to_tt_tensor(tens, max_tt_rank=100, epsilon=0.9) + will probably return you a TT-tensor with TT-ranks close to 1, not 100. + Note that providing a nontrivial (= not equal to None) `epsilon` will make + the TT-ranks of the result undefined on the compilation stage + (e.g. res.get_tt_ranks() will return None, but t3f.tt_ranks(res).eval() + will work). + + Returns: + `TensorTrain` object containing a TT-tensor. + + Raises: + ValueError if the rank (number of dimensions) of the input tensor is + not defined, if max_tt_rank is less than 0, if max_tt_rank is not a number + and not a vector of length d + 1 where d is the number of dimensions (rank) + of the input tensor, if epsilon is less than 0. + """ + tens = tf.convert_to_tensor(tens) + static_shape = tens.get_shape() + dynamic_shape = tf.shape(tens) + # Raises ValueError if ndims is not defined. + d = static_shape.__len__() + max_tt_rank = np.array(max_tt_rank).astype(np.int32) + if max_tt_rank < 1: + raise ValueError('Maximum TT-rank should be greater or equal to 1.') + if epsilon is not None and epsilon < 0: + raise ValueError('Epsilon should be non-negative.') + if max_tt_rank.size == 1: + max_tt_rank = (max_tt_rank * np.ones(d+1)).astype(np.int32) + elif max_tt_rank.size != d + 1: + raise ValueError('max_tt_rank should be a number or a vector of size (d+1) ' + 'where d is the number of dimensions (rank) of the tensor.') + ranks = [1] * (d + 1) + tt_cores = [] + are_tt_ranks_defined = True + for core_idx in range(d - 1): + curr_mode = static_shape[core_idx].value + if curr_mode is None: + curr_mode = dynamic_shape[core_idx] + rows = ranks[core_idx] * curr_mode + tens = tf.reshape(tens, [rows, -1]) + columns = tens.get_shape()[1].value + if columns is None: + columns = tf.shape(tens)[1] + s, u, v = tf.svd(tens, full_matrices=False) + if max_tt_rank[core_idx + 1] == 1: + ranks[core_idx + 1] = 1 + else: + try: + ranks[core_idx + 1] = min(max_tt_rank[core_idx + 1], rows, columns) + except TypeError: + # Some of the values are undefined on the compilation stage and thus + # they are tf.tensors instead of values. + min_dim = tf.minimum(rows, columns) + ranks[core_idx + 1] = tf.minimum(max_tt_rank[core_idx + 1], min_dim) + are_tt_ranks_defined = False + u = u[:, 0:ranks[core_idx + 1]] + s = s[0:ranks[core_idx + 1]] + v = v[:, 0:ranks[core_idx + 1]] + core_shape = (ranks[core_idx], curr_mode, ranks[core_idx + 1]) + tt_cores.append(tf.reshape(u, core_shape)) + tens = tf.matmul(tf.diag(s), tf.transpose(v)) + last_mode = static_shape[-1].value + if last_mode is None: + last_mode = dynamic_shape[-1] + core_shape = (ranks[d - 1], last_mode, ranks[d]) + tt_cores.append(tf.reshape(tens, core_shape)) + if not are_tt_ranks_defined: + ranks = None + return TensorTrain(tt_cores, static_shape, ranks) + + +# TODO: rename round so not to shadow python.round? +def round(tt, max_tt_rank=None, epsilon=None): + """TT-rounding procedure, returns a TT object with smaller TT-ranks. + + Args: + tt: `TensorTrain` object, TT-tensor or TT-matrix + max_tt_rank: a number or a list of numbers + If a number, than defines the maximal TT-rank of the result. + If a list of numbers, than `max_tt_rank` length should be d+1 + (where d is the rank of `tens`) and `max_tt_rank[i]` defines + the maximal (i+1)-th TT-rank of the result. + The following two versions are equivalent + `max_tt_rank = r` + and + `max_tt_rank = r * np.ones(d-1)` + epsilon: a floating point number or None + If the TT-ranks are not restricted (`max_tt_rank=np.inf`), then + the result would be guarantied to be `epsilon` close to `tt` + in terms of relative Frobenius error: + ||res - tt||_F / ||tt||_F <= epsilon + If the TT-ranks are restricted, providing a loose `epsilon` may + reduce the TT-ranks of the result. + E.g. + round(tt, max_tt_rank=100, epsilon=0.9) + will probably return you a TT-tensor with TT-ranks close to 1, not 100. + Note that providing a nontrivial (= not equal to None) `epsilon` will make + the TT-ranks of the result undefined on the compilation stage + (e.g. res.get_tt_ranks() will return None, but t3f.tt_ranks(res).eval() + will work). + + Returns: + `TensorTrain` object containing a TT-tensor. + + Raises: + ValueError if max_tt_rank is less than 0, if max_tt_rank is not a number and + not a vector of length d + 1 where d is the number of dimensions (rank) of + the input tensor, if epsilon is less than 0. + """ + if isinstance(tt, TensorTrainBatch): + return _round_batch_tt(tt, max_tt_rank, epsilon) + else: + return _round_tt(tt, max_tt_rank, epsilon) + + +def _round_tt(tt, max_tt_rank, epsilon): + """Internal function that rounds a TensorTrain (not batch). + + See t3f.round for details. + """ + ndims = tt.ndims() + max_tt_rank = np.array(max_tt_rank).astype(np.int32) + if max_tt_rank < 1: + raise ValueError('Maximum TT-rank should be greater or equal to 1.') + if epsilon is not None and epsilon < 0: + raise ValueError('Epsilon should be non-negative.') + if max_tt_rank.size == 1: + max_tt_rank = (max_tt_rank * np.ones(ndims + 1)).astype(np.int32) + elif max_tt_rank.size != ndims + 1: + raise ValueError('max_tt_rank should be a number or a vector of size (d+1) ' + 'where d is the number of dimensions (rank) of the tensor.') + raw_shape = shapes.lazy_raw_shape(tt) + + tt_cores = orthogonalize_tt_cores(tt).tt_cores + # Copy cores references so we can change the cores. + tt_cores = list(tt_cores) + + ranks = [1] * (ndims + 1) + are_tt_ranks_defined = True + # Right to left SVD compression. + for core_idx in range(ndims - 1, 0, -1): + curr_core = tt_cores[core_idx] + if tt.is_tt_matrix(): + curr_mode_left = raw_shape[0][core_idx] + curr_mode_right = raw_shape[1][core_idx] + curr_mode = curr_mode_left * curr_mode_right + else: + curr_mode = raw_shape[0][core_idx] + + columns = curr_mode * ranks[core_idx + 1] + curr_core = tf.reshape(curr_core, [-1, columns]) + rows = curr_core.get_shape()[0].value + if rows is None: + rows = tf.shape(curr_core)[0] + if max_tt_rank[core_idx] == 1: + ranks[core_idx] = 1 + else: + try: + ranks[core_idx] = min(max_tt_rank[core_idx], rows, columns) + except TypeError: + # Some of the values are undefined on the compilation stage and thus + # they are tf.tensors instead of values. + min_dim = tf.minimum(rows, columns) + ranks[core_idx] = tf.minimum(max_tt_rank[core_idx], min_dim) + are_tt_ranks_defined = False + s, u, v = tf.svd(curr_core, full_matrices=False) + u = u[:, 0:ranks[core_idx]] + s = s[0:ranks[core_idx]] + v = v[:, 0:ranks[core_idx]] + if tt.is_tt_matrix(): + core_shape = (ranks[core_idx], curr_mode_left, curr_mode_right, + ranks[core_idx + 1]) + else: + core_shape = (ranks[core_idx], curr_mode, ranks[core_idx + 1]) + tt_cores[core_idx] = tf.reshape(tf.transpose(v), core_shape) + prev_core_shape = (-1, rows) + tt_cores[core_idx - 1] = tf.reshape(tt_cores[core_idx - 1], prev_core_shape) + tt_cores[core_idx - 1] = tf.matmul(tt_cores[core_idx - 1], u) + tt_cores[core_idx - 1] = tf.matmul(tt_cores[core_idx - 1], tf.diag(s)) + + if tt.is_tt_matrix(): + core_shape = (ranks[0], raw_shape[0][0], raw_shape[1][0], ranks[1]) + else: + core_shape = (ranks[0], raw_shape[0][0], ranks[1]) + tt_cores[0] = tf.reshape(tt_cores[0], core_shape) + if not are_tt_ranks_defined: + ranks = None + return TensorTrain(tt_cores, tt.get_raw_shape(), ranks) + + +def _round_batch_tt(tt, max_tt_rank, epsilon): + """Internal function that rounds a TensorTrainBatch. + + See t3f.round for details. + """ + ndims = tt.ndims() + max_tt_rank = np.array(max_tt_rank).astype(np.int32) + if max_tt_rank < 1: + raise ValueError('Maximum TT-rank should be greater or equal to 1.') + if epsilon is not None and epsilon < 0: + raise ValueError('Epsilon should be non-negative.') + if max_tt_rank.size == 1: + max_tt_rank = (max_tt_rank * np.ones(ndims + 1)).astype(np.int32) + elif max_tt_rank.size != ndims + 1: + raise ValueError('max_tt_rank should be a number or a vector of size (d+1) ' + 'where d is the number of dimensions (rank) of the tensor.') + raw_shape = shapes.lazy_raw_shape(tt) + batch_size = shapes.lazy_batch_size(tt) + + tt_cores = orthogonalize_tt_cores(tt).tt_cores + # Copy cores references so we can change the cores. + tt_cores = list(tt_cores) + + ranks = [1] * (ndims + 1) + are_tt_ranks_defined = True + # Right to left SVD compression. + for core_idx in range(ndims - 1, 0, -1): + curr_core = tt_cores[core_idx] + if tt.is_tt_matrix(): + curr_mode_left = raw_shape[0][core_idx] + curr_mode_right = raw_shape[1][core_idx] + curr_mode = curr_mode_left * curr_mode_right + else: + curr_mode = raw_shape[0][core_idx] + + columns = curr_mode * ranks[core_idx + 1] + curr_core = tf.reshape(curr_core, (batch_size, -1, columns)) + rows = curr_core.get_shape()[1].value + if rows is None: + rows = tf.shape(curr_core)[1] + if max_tt_rank[core_idx] == 1: + ranks[core_idx] = 1 + else: + try: + ranks[core_idx] = min(max_tt_rank[core_idx], rows, columns) + except TypeError: + # Some of the values are undefined on the compilation stage and thus + # they are tf.tensors instead of values. + min_dim = tf.minimum(rows, columns) + ranks[core_idx] = tf.minimum(max_tt_rank[core_idx], min_dim) + are_tt_ranks_defined = False + s, u, v = tf.svd(curr_core, full_matrices=False) + u = u[:, :, 0:ranks[core_idx]] + s = s[:, 0:ranks[core_idx]] + v = v[:, :, 0:ranks[core_idx]] + if tt.is_tt_matrix(): + core_shape = (batch_size, ranks[core_idx], curr_mode_left, curr_mode_right, + ranks[core_idx + 1]) + else: + core_shape = (batch_size, ranks[core_idx], curr_mode, ranks[core_idx + 1]) + tt_cores[core_idx] = tf.reshape(tf.transpose(v, (0, 2, 1)), core_shape) + prev_core_shape = (batch_size, -1, rows) + tt_cores[core_idx - 1] = tf.reshape(tt_cores[core_idx - 1], prev_core_shape) + tt_cores[core_idx - 1] = tf.matmul(tt_cores[core_idx - 1], u) + tt_cores[core_idx - 1] = tf.matmul(tt_cores[core_idx - 1], tf.matrix_diag(s)) + + if tt.is_tt_matrix(): + core_shape = (batch_size, ranks[0], raw_shape[0][0], raw_shape[1][0], ranks[1]) + else: + core_shape = (batch_size, ranks[0], raw_shape[0][0], ranks[1]) + tt_cores[0] = tf.reshape(tt_cores[0], core_shape) + if not are_tt_ranks_defined: + ranks = None + return TensorTrainBatch(tt_cores, tt.get_raw_shape(), ranks, batch_size=tt.batch_size) + + +def orthogonalize_tt_cores(tt, left_to_right=True): + """Orthogonalize TT-cores of a TT-object. + + Args: + tt: TenosorTrain or a TensorTrainBatch. + left_to_right: bool, the direction of orthogonalization. + + Returns: + The same type as the input `tt` (TenosorTrain or a TensorTrainBatch). + """ + if isinstance(tt, TensorTrainBatch): + if left_to_right: + return _orthogonalize_batch_tt_cores_left_to_right(tt) + else: + raise NotImplementedError('Batch right to left orthogonalization is not ' + 'supported yet.') + else: + if left_to_right: + return _orthogonalize_tt_cores_left_to_right(tt) + else: + return _orthogonalize_tt_cores_right_to_left(tt) + + +def _orthogonalize_tt_cores_left_to_right(tt): + """Orthogonalize TT-cores of a TT-object in the left to right order. + Args: + tt: TenosorTrain or a TensorTrainBatch. + Returns: + The same type as the input `tt` (TenosorTrain or a TensorTrainBatch). + + Complexity: + for a single TT-object: + O(d r^3 n) + for a batch of TT-objects: + O(batch_size d r^3 n) + where + d is the number of TT-cores (tt.ndims()); + r is the largest TT-rank of tt max(tt.get_tt_rank()) + n is the size of the axis dimension, e.g. + for a tensor of size 4 x 4 x 4, n is 4; + for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 + """ + # Left to right orthogonalization. + ndims = tt.ndims() + raw_shape = shapes.lazy_raw_shape(tt) + tt_ranks = shapes.lazy_tt_ranks(tt) + next_rank = tt_ranks[0] + # Copy cores references so we can change the cores. + tt_cores = list(tt.tt_cores) + for core_idx in range(ndims - 1): + curr_core = tt_cores[core_idx] + # TT-ranks could have changed on the previous iteration, so `tt_ranks` can + # be outdated for the current TT-rank, but should be valid for the next + # TT-rank. + curr_rank = next_rank + next_rank = tt_ranks[core_idx + 1] + if tt.is_tt_matrix(): + curr_mode_left = raw_shape[0][core_idx] + curr_mode_right = raw_shape[1][core_idx] + curr_mode = curr_mode_left * curr_mode_right + else: + curr_mode = raw_shape[0][core_idx] + + qr_shape = (curr_rank * curr_mode, next_rank) + curr_core = tf.reshape(curr_core, qr_shape) + curr_core, triang = tf.qr(curr_core) + if triang.get_shape().is_fully_defined(): + triang_shape = triang.get_shape().as_list() + else: + triang_shape = tf.shape(triang) + # The TT-rank could have changed: if qr_shape is e.g. 4 x 10, than q would + # be of size 4 x 4 and r would be 4 x 10, which means that the next rank + # should be changed to 4. + next_rank = triang_shape[0] + if tt.is_tt_matrix(): + new_core_shape = (curr_rank, curr_mode_left, curr_mode_right, next_rank) + else: + new_core_shape = (curr_rank, curr_mode, next_rank) + tt_cores[core_idx] = tf.reshape(curr_core, new_core_shape) + + next_core = tf.reshape(tt_cores[core_idx + 1], (triang_shape[1], -1)) + tt_cores[core_idx + 1] = tf.matmul(triang, next_core) + + if tt.is_tt_matrix(): + last_core_shape = (next_rank, raw_shape[0][-1], raw_shape[1][-1], 1) + else: + last_core_shape = (next_rank, raw_shape[0][-1], 1) + tt_cores[-1] = tf.reshape(tt_cores[-1], last_core_shape) + # TODO: infer the tt_ranks. + return TensorTrain(tt_cores, tt.get_raw_shape()) + + +def _orthogonalize_batch_tt_cores_left_to_right(tt): + """Orthogonalize TT-cores of a batch TT-object in the left to right order. + + Args: + tt: TensorTrainBatch. + + Returns: + TensorTrainBatch + """ + # Left to right orthogonalization. + ndims = tt.ndims() + raw_shape = shapes.lazy_raw_shape(tt) + tt_ranks = shapes.lazy_tt_ranks(tt) + next_rank = tt_ranks[0] + batch_size = shapes.lazy_batch_size(tt) + + # Copy cores references so we can change the cores. + tt_cores = list(tt.tt_cores) + for core_idx in range(ndims - 1): + curr_core = tt_cores[core_idx] + # TT-ranks could have changed on the previous iteration, so `tt_ranks` can + # be outdated for the current TT-rank, but should be valid for the next + # TT-rank. + curr_rank = next_rank + next_rank = tt_ranks[core_idx + 1] + if tt.is_tt_matrix(): + curr_mode_left = raw_shape[0][core_idx] + curr_mode_right = raw_shape[1][core_idx] + curr_mode = curr_mode_left * curr_mode_right + else: + curr_mode = raw_shape[0][core_idx] + + qr_shape = (batch_size, curr_rank * curr_mode, next_rank) + curr_core = tf.reshape(curr_core, qr_shape) + curr_core, triang = tf.qr(curr_core) + if triang.get_shape().is_fully_defined(): + triang_shape = triang.get_shape().as_list() + else: + triang_shape = tf.shape(triang) + # The TT-rank could have changed: if qr_shape is e.g. 4 x 10, than q would + # be of size 4 x 4 and r would be 4 x 10, which means that the next rank + # should be changed to 4. + next_rank = triang_shape[1] + if tt.is_tt_matrix(): + new_core_shape = (batch_size, curr_rank, curr_mode_left, curr_mode_right, + next_rank) + else: + new_core_shape = (batch_size, curr_rank, curr_mode, next_rank) + + tt_cores[core_idx] = tf.reshape(curr_core, new_core_shape) + + next_core = tf.reshape(tt_cores[core_idx + 1], (batch_size, triang_shape[2], -1)) + tt_cores[core_idx + 1] = tf.matmul(triang, next_core) + + if tt.is_tt_matrix(): + last_core_shape = (batch_size, next_rank, raw_shape[0][-1], + raw_shape[1][-1], 1) + else: + last_core_shape = (batch_size, next_rank, raw_shape[0][-1], 1) + tt_cores[-1] = tf.reshape(tt_cores[-1], last_core_shape) + # TODO: infer the tt_ranks. + return TensorTrainBatch(tt_cores, tt.get_raw_shape(), batch_size=batch_size) + + +def _orthogonalize_tt_cores_right_to_left(tt): + """Orthogonalize TT-cores of a TT-object in the right to left order. + + Args: + tt: TenosorTrain or a TensorTrainBatch. + + Returns: + The same type as the input `tt` (TenosorTrain or a TensorTrainBatch). + """ + # Left to right orthogonalization. + ndims = tt.ndims() + raw_shape = shapes.lazy_raw_shape(tt) + tt_ranks = shapes.lazy_tt_ranks(tt) + prev_rank = tt_ranks[ndims] + # Copy cores references so we can change the cores. + tt_cores = list(tt.tt_cores) + for core_idx in range(ndims - 1, 0, -1): + curr_core = tt_cores[core_idx] + # TT-ranks could have changed on the previous iteration, so `tt_ranks` can + # be outdated for the current TT-rank, but should be valid for the next + # TT-rank. + curr_rank = prev_rank + prev_rank = tt_ranks[core_idx] + if tt.is_tt_matrix(): + curr_mode_left = raw_shape[0][core_idx] + curr_mode_right = raw_shape[1][core_idx] + curr_mode = curr_mode_left * curr_mode_right + else: + curr_mode = raw_shape[0][core_idx] + + qr_shape = (prev_rank, curr_mode * curr_rank) + curr_core = tf.reshape(curr_core, qr_shape) + curr_core, triang = tf.qr(tf.transpose(curr_core)) + curr_core = tf.transpose(curr_core) + triang = tf.transpose(triang) + if triang.get_shape().is_fully_defined(): + triang_shape = triang.get_shape().as_list() + else: + triang_shape = tf.shape(triang) + # The TT-rank could have changed: if qr_shape is e.g. 4 x 10, than q would + # be of size 4 x 4 and r would be 4 x 10, which means that the next rank + # should be changed to 4. + prev_rank = triang_shape[1] + if tt.is_tt_matrix(): + new_core_shape = (prev_rank, curr_mode_left, curr_mode_right, curr_rank) + else: + new_core_shape = (prev_rank, curr_mode, curr_rank) + tt_cores[core_idx] = tf.reshape(curr_core, new_core_shape) + + prev_core = tf.reshape(tt_cores[core_idx - 1], (-1, triang_shape[0])) + tt_cores[core_idx - 1] = tf.matmul(prev_core, triang) + + if tt.is_tt_matrix(): + first_core_shape = (1, raw_shape[0][0], raw_shape[1][0], prev_rank) + else: + first_core_shape = (1, raw_shape[0][0], prev_rank) + tt_cores[0] = tf.reshape(tt_cores[0], first_core_shape) + # TODO: infer the tt_ranks. + return TensorTrain(tt_cores, tt.get_raw_shape()) diff --git a/build/lib/t3f/decompositions_test.py b/build/lib/t3f/decompositions_test.py new file mode 100644 index 00000000..158eff9e --- /dev/null +++ b/build/lib/t3f/decompositions_test.py @@ -0,0 +1,176 @@ +import numpy as np +import tensorflow as tf + +from t3f import ops +from t3f import shapes +from t3f import decompositions +from t3f import initializers + + +class DecompositionsTest(tf.test.TestCase): + + def testTTTensor(self): + shape = (2, 1, 4, 3) + np.random.seed(1) + tens = np.random.rand(*shape).astype(np.float32) + tf_tens = tf.constant(tens) + tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=3) + with self.test_session(): + self.assertAllClose(tens, ops.full(tt_tens).eval()) + dynamic_tt_ranks = shapes.tt_ranks(tt_tens).eval() + static_tt_ranks = tt_tens.get_tt_ranks().as_list() + self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) + + # Try to decompose the same tensor with unknown shape. + tf_tens_pl = tf.placeholder(tf.float32, (None, None, 4, None)) + tt_tens = decompositions.to_tt_tensor(tf_tens_pl, max_tt_rank=3) + tt_val = ops.full(tt_tens).eval({tf_tens_pl: tens}) + self.assertAllClose(tens, tt_val) + dynamic_tt_ranks = shapes.tt_ranks(tt_tens).eval({tf_tens_pl: tens}) + self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) + + def testTTTensorSimple(self): + # Test that a tensor of ones and of zeros can be converted into TT with + # TT-rank 1. + shape = (2, 1, 4, 3) + tens_arr = (np.zeros(shape).astype(np.float32), + np.ones(shape).astype(np.float32)) + for tens in tens_arr: + tf_tens = tf.constant(tens) + tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=1) + with self.test_session(): + self.assertAllClose(tens, ops.full(tt_tens).eval()) + dynamic_tt_ranks = shapes.tt_ranks(tt_tens).eval() + static_tt_ranks = tt_tens.get_tt_ranks().as_list() + self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) + + # Try to decompose the same tensor with unknown shape. + tf_tens_pl = tf.placeholder(tf.float32, (None, None, None, None)) + tt_tens = decompositions.to_tt_tensor(tf_tens_pl, max_tt_rank=1) + tt_val = ops.full(tt_tens).eval({tf_tens_pl: tens}) + self.assertAllClose(tens, tt_val) + dynamic_tt_ranks = shapes.tt_ranks(tt_tens).eval({tf_tens_pl: tens}) + self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) + + def testTTVector(self): + vec_shape = (2, 1, 4, 3) + np.random.seed(1) + rows = np.prod(vec_shape) + vec = np.random.rand(rows, 1).astype(np.float32) + tf_vec = tf.constant(vec) + tt_vec = decompositions.to_tt_matrix(tf_vec, (vec_shape, None)) + with self.test_session(): + self.assertAllClose(vec, ops.full(tt_vec).eval()) + + def testTTMatrix(self): + # Convert a np.prod(out_shape) x np.prod(in_shape) matrix into TT-matrix + # and back. + inp_shape = (2, 5, 2, 3) + out_shape = (3, 3, 2, 3) + np.random.seed(1) + mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) + mat = mat.astype(np.float32) + tf_mat = tf.constant(mat) + tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), + max_tt_rank=90) + with self.test_session(): + # TODO: why so bad accuracy? + self.assertAllClose(mat, ops.full(tt_mat).eval(), atol=1e-5, rtol=1e-5) + + def testRoundTensor(self): + shape = (2, 1, 4, 3, 3) + np.random.seed(1) + tens = initializers.random_tensor(shape, tt_rank=15) + rounded_tens = decompositions.round(tens, max_tt_rank=9) + with self.test_session() as sess: + vars = [ops.full(tens), ops.full(rounded_tens)] + tens_value, rounded_tens_value = sess.run(vars) + # TODO: why so bad accuracy? + self.assertAllClose(tens_value, rounded_tens_value, atol=1e-4, rtol=1e-4) + dynamic_tt_ranks = shapes.tt_ranks(rounded_tens).eval() + self.assertAllEqual([1, 2, 2, 8, 3, 1], dynamic_tt_ranks) + + def testOrthogonalizeLeftToRight(self): + shape = (2, 4, 3, 3) + tt_ranks = (1, 5, 2, 17, 1) + updated_tt_ranks = (1, 2, 2, 6, 1) + tens = initializers.random_tensor(shape, tt_rank=tt_ranks) + orthogonal = decompositions.orthogonalize_tt_cores(tens) + with self.test_session() as sess: + tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) + self.assertAllClose(tens_val, orthogonal_val, atol=1e-5, rtol=1e-5) + dynamic_tt_ranks = shapes.tt_ranks(orthogonal).eval() + self.assertAllEqual(updated_tt_ranks, dynamic_tt_ranks) + # Check that the TT-cores are orthogonal. + for core_idx in range(4 - 1): + core = orthogonal.tt_cores[core_idx] + core = tf.reshape(core, (updated_tt_ranks[core_idx] * shape[core_idx], + updated_tt_ranks[core_idx + 1])) + should_be_eye = tf.matmul(tf.transpose(core), core) + should_be_eye_val = sess.run(should_be_eye) + self.assertAllClose(np.eye(updated_tt_ranks[core_idx + 1]), + should_be_eye_val) + + def testOrthogonalizeRightToLeft(self): + shape = (2, 4, 3, 3) + tt_ranks = (1, 5, 2, 17, 1) + updated_tt_ranks = (1, 5, 2, 3, 1) + tens = initializers.random_tensor(shape, tt_rank=tt_ranks) + orthogonal = decompositions.orthogonalize_tt_cores(tens, left_to_right=False) + with self.test_session() as sess: + tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) + self.assertAllClose(tens_val, orthogonal_val, atol=1e-5, rtol=1e-5) + dynamic_tt_ranks = shapes.tt_ranks(orthogonal).eval() + self.assertAllEqual(updated_tt_ranks, dynamic_tt_ranks) + # Check that the TT-cores are orthogonal. + for core_idx in range(1, 4): + core = orthogonal.tt_cores[core_idx] + core = tf.reshape(core, (updated_tt_ranks[core_idx], shape[core_idx] * + updated_tt_ranks[core_idx + 1])) + should_be_eye = tf.matmul(core, tf.transpose(core)) + should_be_eye_val = sess.run(should_be_eye) + self.assertAllClose(np.eye(updated_tt_ranks[core_idx]), + should_be_eye_val) + + +class DecompositionsBatchTest(tf.test.TestCase): + + def testOrthogonalizeLeftToRight(self): + shape = (2, 4, 3, 3) + tt_ranks = (1, 5, 2, 17, 1) + updated_tt_ranks = (1, 2, 2, 6, 1) + tens = initializers.random_tensor_batch(shape, tt_rank=tt_ranks, + batch_size=2) + orthogonal = decompositions.orthogonalize_tt_cores(tens) + with self.test_session() as sess: + tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) + self.assertAllClose(tens_val, orthogonal_val, atol=1e-5, rtol=1e-5) + dynamic_tt_ranks = shapes.tt_ranks(orthogonal).eval() + self.assertAllEqual(updated_tt_ranks, dynamic_tt_ranks) + # Check that the TT-cores are orthogonal. + for core_idx in range(4 - 1): + core_shape = (updated_tt_ranks[core_idx] * shape[core_idx], + updated_tt_ranks[core_idx + 1]) + for i in range(2): + core = tf.reshape(orthogonal.tt_cores[core_idx][i], core_shape) + should_be_eye = tf.matmul(tf.transpose(core), core) + should_be_eye_val = sess.run(should_be_eye) + self.assertAllClose(np.eye(updated_tt_ranks[core_idx + 1]), + should_be_eye_val) + + def testRoundTensor(self): + shape = (2, 1, 4, 3, 3) + tens = initializers.random_tensor_batch(shape, tt_rank=15, batch_size=3) + rounded_tens = decompositions.round(tens, max_tt_rank=9) + with self.test_session() as sess: + vars = [ops.full(tens), ops.full(rounded_tens)] + tens_value, rounded_tens_value = sess.run(vars) + # TODO: why so bad accuracy? + self.assertAllClose(tens_value, rounded_tens_value, atol=1e-4, + rtol=1e-4) + dynamic_tt_ranks = shapes.tt_ranks(rounded_tens).eval() + self.assertAllEqual([1, 2, 2, 8, 3, 1], dynamic_tt_ranks) + + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/examples_tests.py b/build/lib/t3f/examples_tests.py new file mode 100644 index 00000000..63dbc151 --- /dev/null +++ b/build/lib/t3f/examples_tests.py @@ -0,0 +1,43 @@ +"""Tests from the README examples and the paper.""" + +import tensorflow as tf +import t3f + +class ExamplesTest(tf.test.TestCase): + + def testMainReadme(self): + # Just check that the readme examples do not raise exceptions. + # Create a random tensor of shape (3, 2, 2). + a = t3f.random_tensor((3, 2, 2), tt_rank=3) + norm = t3f.frobenius_norm(a) + # Convert TT-tensor into a dense tensor for printing. + a_full = t3f.full(a) + # Run a tensorflow session to run the operations. + with tf.Session() as sess: + # Run the operations. Note that if you run these + # two operations separetly (sess.run(a_full), sess.run(norm)) + # the result will be different, since sess.run will + # generate a new random tensor a on each run because `a' is + # an operation 'generate me a random tensor'. + a_val, norm_val = sess.run([a_full, norm]) + a = t3f.random_tensor((3, 2, 2), tt_rank=3) + b_dense = tf.random_normal((3, 2, 2)) + # Use TT-SVD on b_dense. + b_tt = t3f.to_tt_tensor(b_dense, max_tt_rank=4) + sum_round = t3f.round(t3f.add(a, b_tt), max_tt_rank=2) + # Inner product (sum of products of all elements). + a = t3f.random_tensor((3, 2, 2), tt_rank=3) + b = t3f.random_tensor((3, 2, 2), tt_rank=4) + inner_prod = t3f.flat_inner(a, b) + A = t3f.random_matrix(((3, 2, 2), (2, 3, 3)), tt_rank=3) + b = t3f.random_matrix(((2, 3, 3), None), tt_rank=3) + # Matrix-by-vector + matvec = t3f.matmul(A, b) + + # Matrix-by-dense matrix + b_dense = tf.random_normal((18, 1)) + matvec2 = t3f.matmul(A, b_dense) + + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/initializers.py b/build/lib/t3f/initializers.py new file mode 100644 index 00000000..6f192cc0 --- /dev/null +++ b/build/lib/t3f/initializers.py @@ -0,0 +1,789 @@ +import numpy as np +import tensorflow as tf + +from t3f.tensor_train import TensorTrain +from t3f.tensor_train_batch import TensorTrainBatch +from t3f.tensor_train_base import TensorTrainBase +from t3f import shapes + + +def _validate_input_parameters(is_tensor, shape, **params): + """Internal function for validating input parameters + + Args: + is_tensor: bool, determines whether we attempt to construct a TT-tensor or + a TT-matrix (needed for the correct shape checks). + shape: array, the desired shape of the generated TT object + params: optional, possible values: + batch_size: int, for constructing batches + tt_rank: array or int, desired TT-ranks + """ + + if is_tensor: + if len(shape.shape) != 1: + raise ValueError('shape should be 1d array, got %a' % shape) + if np.any(shape < 1): + raise ValueError('all elements in `shape` should be positive, got %a' % + shape) + if not all(isinstance(sh, np.integer) for sh in shape): + raise ValueError('all elements in `shape` should be integers, got %a' % + shape) + else: + if len(shape.shape) != 2: + raise ValueError('shape should be 2d array, got %a' % shape) + if shape[0].size != shape[1].size: + raise ValueError('shape[0] should have the same length as shape[1], but' + 'got %d and %d' % (shape[0].size, shape[1].size)) + if np.any(shape.flatten() < 1): + raise ValueError('all elements in `shape` should be positive, got %a' % + shape) + if not all(isinstance(sh, np.integer) for sh in shape.flatten()): + raise ValueError('all elements in `shape` should be integers, got %a' % + shape) + + if 'batch_size' in params: + batch_size = params['batch_size'] + if not isinstance(batch_size, (int, np.integer)): + raise ValueError('`batch_size` should be integer, got %f' % batch_size) + if batch_size < 1: + raise ValueError('Batch size should be positive, got %d' % batch_size) + if 'tt_rank' in params: + tt_rank = params['tt_rank'] + if tt_rank.size == 1: + if not isinstance(tt_rank[()], np.integer): + raise ValueError('`tt_rank` should be integer, got %f' % tt_rank[()]) + if tt_rank.size > 1: + if not all(isinstance(tt_r, np.integer) for tt_r in tt_rank): + raise ValueError('all elements in `tt_rank` should be integers, got' + ' %a' % tt_rank) + if np.any(tt_rank < 1): + raise ValueError('`tt_rank` should be positive, got %a' % tt_rank) + + if is_tensor: + if tt_rank.size != 1 and tt_rank.size != (shape.size + 1): + raise ValueError('`tt_rank` array has inappropriate size, expected' + '1 or %d, got %d' % (shape.size + 1, tt_rank.size)) + else: + if tt_rank.size != 1 and tt_rank.size != (shape[0].size + 1): + raise ValueError('`tt_rank` array has inappropriate size, expected' + '1 or %d, got %d' % (shape[0].size + 1, tt_rank.size)) + + +def tensor_ones(shape): + """Generate TT-tensor of the given shape with all entries equal to 1. + + Args: + shape: array representing the shape of the future tensor + + Returns: + TensorTrain object containing a TT-tensor + """ + + shape = np.array(shape) + _validate_input_parameters(is_tensor=True, shape=shape) + num_dims = shape.size + tt_rank = np.ones(num_dims + 1) + + tt_cores = num_dims * [None] + for i in range(num_dims): + curr_core_shape = (1, shape[i], 1) + tt_cores[i] = tf.ones(curr_core_shape) + + return TensorTrain(tt_cores, shape, tt_rank) + + +def tensor_zeros(shape): + """Generate TT-tensor of the given shape with all entries equal to 0. + + Args: + shape: array representing the shape of the future tensor + + Returns: + TensorTrain object containing a TT-tensor + """ + + shape = np.array(shape) + _validate_input_parameters(is_tensor=True, shape=shape) + num_dims = shape.size + tt_rank = np.ones(num_dims + 1) + tt_cores = num_dims * [None] + for i in range(num_dims): + curr_core_shape = (1, shape[i], 1) + tt_cores[i] = tf.zeros(curr_core_shape) + + return TensorTrain(tt_cores, shape, tt_rank) + + +def eye(shape): + """Creates an identity TT-matrix. + + Args: + shape: array which defines the shape of the matrix row and column + indices. + + Returns: + TensorTrain containing an identity TT-matrix of size + np.prod(shape) x np.prod(shape) + """ + shape = np.array(shape) + # In this special case shape is in the same format as in the TT-tensor case + _validate_input_parameters(is_tensor=True, shape=shape) + + num_dims = shape.size + tt_ranks = np.ones(num_dims + 1) + + tt_cores = num_dims * [None] + for i in range(num_dims): + curr_core_shape = (1, shape[i], shape[i], 1) + tt_cores[i] = tf.reshape(tf.eye(shape[i]), curr_core_shape) + + true_shape = np.vstack([shape, shape]) + return TensorTrain(tt_cores, true_shape, tt_ranks) + + +def matrix_ones(shape): + """Generate a TT-matrix of the given shape with each entry equal to 1. + + Args: + shape: 2d array, shape[0] is the shape of the matrix row-index, + shape[1] is the shape of the column index. + shape[0] and shape[1] should have the same number of elements (d) + Also supports omitting one of the dimensions for vectors, e.g. + matrix_ones([[2, 2, 2], None]) + and + matrix_ones([None, [2, 2, 2]]) + will create an 8-element column and row vectors correspondingly. + + Returns: + TensorTrain containing a TT-matrix of size + np.prod(shape[0]) x np.prod(shape[1]) with each entry equal to 1 + """ + + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1]), dtype=int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0]), dtype=int) + shape = np.array(shape) + + _validate_input_parameters(is_tensor=False, shape=shape) + + num_dims = shape[0].size + tt_rank = np.ones(shape[0].size + 1) + + # TODO: variable (name?) scope. + + tt_cores = [None] * num_dims + for i in range(num_dims): + curr_core_shape = (1, shape[0][i], shape[1][i], 1) + tt_cores[i] = tf.ones(curr_core_shape) + + return TensorTrain(tt_cores, shape, tt_rank) + + +def matrix_zeros(shape): + """Generate a TT-matrix of the given shape with each entry equal to 0. + + Args: + shape: 2d array, shape[0] is the shape of the matrix row-index, + shape[1] is the shape of the column index. + shape[0] and shape[1] should have the same number of elements (d) + Also supports omitting one of the dimensions for vectors, e.g. + matrix_zeros([[2, 2, 2], None]) + and + matrix_zeros([None, [2, 2, 2]]) + will create an 8-element column and row vectors correspondingly. + + Returns: + TensorTrain containing a TT-matrix of size + np.prod(shape[0]) x np.prod(shape[1]) with each entry equal to 0 + """ + + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1]), dtype=int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0]), dtype=int) + shape = np.array(shape) + + _validate_input_parameters(is_tensor=False, shape=shape) + num_dims = shape[0].size + tt_rank = np.ones(shape[0].size + 1) + + # TODO: variable (name?) scope. + + tt_cores = [None] * num_dims + for i in range(num_dims): + curr_core_shape = (1, shape[0][i], shape[1][i], 1) + tt_cores[i] = tf.zeros(curr_core_shape) + + return TensorTrain(tt_cores, shape, tt_rank) + + +def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): + """Generate a TT-tensor of the given shape with N(mean, stddev^2) cores. + + Args: + shape: array representing the shape of the future tensor. + tt_rank: a number or a (d+1)-element array with the desired ranks. + mean: a number, the mean of the normal distribution used for + initializing TT-cores. + stddev: a number, the standard deviation of the normal distribution used + for initializing TT-cores. + + Returns: + TensorTrain containing a TT-tensor + """ + # TODO: good distribution to init training. + # TODO: support shape and tt_ranks as TensorShape?. + # TODO: support None as a dimension. + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=True, shape=shape, tt_rank=tt_rank) + num_dims = shape.size + if tt_rank.size == 1: + tt_rank = tt_rank * np.ones(num_dims - 1) + tt_rank = np.insert(tt_rank, 0, 1) + tt_rank = np.append(tt_rank, 1) + + tt_rank = tt_rank.astype(int) + # TODO: variable (name?) scope. + tt_cores = [None] * num_dims + for i in range(num_dims): + curr_core_shape = (tt_rank[i], shape[i], tt_rank[i + 1]) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + + return TensorTrain(tt_cores, shape, tt_rank) + + +def tensor_batch_with_random_cores(shape, tt_rank=2, batch_size=1, + mean=0., stddev=1.): + """Generate a batch of TT-tensors of given shape with N(mean, stddev^2) cores. + + Args: + shape: array representing the shape of the future tensor. + tt_rank: a number or a (d+1)-element array with ranks. + batch_size: an integer. + mean: a number, the mean of the normal distribution used for + initializing TT-cores. + stddev: a number, the standard deviation of the normal distribution used + for initializing TT-cores. + + Returns: + TensorTrainBatch containing TT-tensors + """ + + # TODO: support shape and tt_ranks as TensorShape?. + # TODO: support None as a dimension. + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=True, shape=shape, tt_rank=tt_rank, + batch_size=batch_size) + num_dims = shape.size + if tt_rank.size == 1: + tt_rank = tt_rank * np.ones(num_dims - 1) + tt_rank = np.insert(tt_rank, 0, 1) + tt_rank = np.append(tt_rank, 1) + tt_rank = tt_rank.astype(int) + # TODO: variable (name?) scope. + tt_cores = [None] * num_dims + for i in range(num_dims): + curr_core_shape = (batch_size, tt_rank[i], shape[i], tt_rank[i + 1]) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + + return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) + + +def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): + """Generate a TT-matrix of given shape with N(mean, stddev^2) cores. + + Args: + shape: 2d array, shape[0] is the shape of the matrix row-index, + shape[1] is the shape of the column index. + shape[0] and shape[1] should have the same number of elements (d) + Also supports omitting one of the dimensions for vectors, e.g. + matrix_with_random_cores([[2, 2, 2], None]) + and + matrix_with_random_cores([None, [2, 2, 2]]) + will create an 8-element column and row vectors correspondingly. + tt_rank: a number or a (d+1)-element array with ranks. + mean: a number, the mean of the normal distribution used for + initializing TT-cores. + stddev: a number, the standard deviation of the normal distribution used + for initializing TT-cores. + + Returns: + TensorTrain containing a TT-matrix of size + np.prod(shape[0]) x np.prod(shape[1]) + """ + # TODO: good distribution to init training. + # In case the shape is immutable. + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1]), dtype=int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0]), dtype=int) + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) + + num_dims = shape[0].size + if tt_rank.size == 1: + tt_rank = tt_rank * np.ones(num_dims - 1) + tt_rank = np.concatenate([[1], tt_rank, [1]]) + + tt_rank = tt_rank.astype(int) + # TODO: variable (name?) scope. + tt_cores = [None] * num_dims + for i in range(num_dims): + curr_core_shape = (tt_rank[i], shape[0][i], shape[1][i], + tt_rank[i + 1]) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + + return TensorTrain(tt_cores, shape, tt_rank) + + +def matrix_batch_with_random_cores(shape, tt_rank=2, batch_size=1, + mean=0., stddev=1.): + """Generate a batch of TT-matrices of given shape with N(mean, stddev^2) cores. + + Args: + shape: 2d array, shape[0] is the shape of the matrix row-index, + shape[1] is the shape of the column index. + shape[0] and shape[1] should have the same number of elements (d) + Also supports omitting one of the dimensions for vectors, e.g. + matrix_batch_with_random_cores([[2, 2, 2], None]) + and + matrix_batch_with_random_cores([None, [2, 2, 2]]) + will create a batch of one 8-element column and row vector correspondingly. + + tt_rank: a number or a (d+1)-element array with ranks. + batch_size: an integer. + mean: a number, the mean of the normal distribution used for + initializing TT-cores. + stddev: a number, the standard deviation of the normal distribution used + for initializing TT-cores. + + Returns: + TensorTrainBatch containing a batch of TT-matrices of size + np.prod(shape[0]) x np.prod(shape[1]) + """ + # TODO: good distribution to init training. + # In case the shape is immutable. + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1]), dtype=int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0]), dtype=int) + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank, + batch_size=batch_size) + num_dims = shape[0].size + if tt_rank.size == 1: + tt_rank = tt_rank * np.ones(num_dims - 1) + tt_rank = np.concatenate([[1], tt_rank, [1]]) + shape = shape.astype(int) + tt_rank = tt_rank.astype(int) + # TODO: variable (name?) scope. + tt_cores = [None] * num_dims + for i in range(num_dims): + curr_core_shape = (batch_size, tt_rank[i], shape[0][i], shape[1][i], + tt_rank[i + 1]) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + + return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) + + +def ones_like(tt): + """Constructs t3f.ones with the shape of `tt`. + + In the case when `tt` is TensorTrainBatch constructs t3f.ones with the shape + of a TensorTrain in `tt`. + + Args: + tt: TensorTrain object + + Returns: + TensorTrain object of the same shape as `tt` but with all entries equal to + 1. + + """ + if not isinstance(tt, TensorTrainBase): + raise ValueError("`tt` has to be a Tensor Train object") + else: + shape = shapes.lazy_raw_shape(tt) + if tt.is_tt_matrix(): + return matrix_ones(shape) + else: + return tensor_ones(shape[0, :]) + + +def zeros_like(tt): + """Constructs t3f.zeros with the shape of `tt`. + + In the case when `tt` is a TensorTrainBatch constructs t3f.zeros with + the shape of a TensorTrain in `tt`. + + Args: + tt: TensorTrain object + + Returns: + TensorTrain object of the same shape as `tt` but with all entries equal to + 0. + + """ + if not isinstance(tt, TensorTrainBase): + raise ValueError("`tt` has to be a Tensor Train object") + else: + shape = shapes.lazy_raw_shape(tt) + if tt.is_tt_matrix(): + return matrix_zeros(shape) + else: + return tensor_zeros(shape[0, :]) + + +def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): + """Generate a random TT-tensor of the given shape with given mean and stddev. + + Entries of the generated tensor (in the full format) will be iid and satisfy + E[x_{i1i2..id}] = mean, Var[x_{i1i2..id}] = stddev^2, but the distribution is + in fact not Gaussian (but is close for large tensors). + + In the current implementation only mean 0 is supported. To get + a random_tensor with specified mean but tt_rank greater by 1 you can + call + x = t3f.random_tensor(shape, tt_rank, stddev=stddev) + x = mean * t3f.ones_like(x) + x + + Args: + shape: array representing the shape of the future tensor. + tt_rank: a number or a (d+1)-element array with the desired ranks. + mean: a number, the desired mean for the distribution of entries. + stddev: a number, the desired standard deviation for the distribution of + entries. + + Returns: + TensorTrain containing a TT-tensor + """ + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=True, shape=shape, tt_rank=tt_rank) + + num_dims = shape.size + if tt_rank.size == 1: + tt_rank = tt_rank * np.ones(num_dims - 1) + tt_rank = np.insert(tt_rank, 0, 1) + tt_rank = np.append(tt_rank, 1) + + tt_rank = tt_rank.astype(int) + + # Empirically entries of a TT tensor with cores initialized from N(0, 1) + # will have variances np.prod(tt_rank) and mean 0. + # We scale each TT-core to obtain the desired stddev + + cr_exponent = -1.0 / (2 * num_dims) + var = np.prod(tt_rank ** cr_exponent) + core_stddev = stddev ** (1.0 / num_dims) * var + tt = tensor_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) + + if np.abs(mean) < 1e-8: + return tt + else: + raise NotImplementedError('non-zero mean is not supported yet') + + +def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): + """Generate a batch of TT-tensors with given shape, mean and stddev. + + Entries of the generated tensors (in the full format) will be iid and satisfy + E[x_{i1i2..id}] = mean, Var[x_{i1i2..id}] = stddev^2, but the distribution is + in fact not Gaussian (but is close for large tensors). + + In the current implementation only mean 0 is supported. To get + a random_tensor_batch with specified mean but tt_rank greater by 1 you can + call + x = t3f.random_tensor_batch(shape, tt_rank, batch_size=bs, stddev=stddev) + x = mean * t3f.ones_like(x) + x + + Args: + shape: array representing the shape of the future tensor. + tt_rank: a number or a (d+1)-element array with ranks. + batch_size: an integer. + mean: a number, the desired mean for the distribution of entries. + stddev: a number, the desired standard deviation for the distribution of + entries. + + Returns: + TensorTrainBatch containing TT-tensors. + """ + # TODO: support shape and tt_ranks as TensorShape?. + # TODO: support None as a dimension. + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=True, shape=shape, tt_rank=tt_rank, + batch_size=batch_size) + num_dims = shape.size + if tt_rank.size == 1: + tt_rank = tt_rank * np.ones(num_dims - 1) + tt_rank = np.insert(tt_rank, 0, 1) + tt_rank = np.append(tt_rank, 1) + tt_rank = tt_rank.astype(int) + + cr_exponent = -1.0 / (2 * num_dims) + var = np.prod(tt_rank ** cr_exponent) + cr_stddev = stddev ** (1.0 / num_dims) * var + tt = tensor_batch_with_random_cores(shape, tt_rank=tt_rank, stddev=cr_stddev, + batch_size=batch_size) + + if np.abs(mean) < 1e-8: + return tt + else: + raise NotImplementedError('non-zero mean is not supported yet') + + +def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): + """Generate a random TT-matrix of the given shape with given mean and stddev. + + Entries of the generated matrix (in the full format) will be iid and satisfy + E[x_{i1i2..id}] = mean, Var[x_{i1i2..id}] = stddev^2, but the distribution is + in fact not Gaussian. + + In the current implementation only mean 0 is supported. To get + a random_matrix with specified mean but tt_rank greater by 1 you can call + x = t3f.random_matrix(shape, tt_rank, stddev=stddev) + x = mean * t3f.ones_like(x) + x + + Args: + shape: 2d array, shape[0] is the shape of the matrix row-index, + shape[1] is the shape of the column index. + shape[0] and shape[1] should have the same number of elements (d) + Also supports omitting one of the dimensions for vectors, e.g. + random_matrix([[2, 2, 2], None]) + and + random_matrix([None, [2, 2, 2]]) + will create an 8-element column and row vectors correspondingly. + tt_rank: a number or a (d+1)-element array with ranks. + mean: a number, the desired mean for the distribution of entries. + stddev: a number, the desired standard deviation for the distribution of + entries. + + Returns: + TensorTrain containing a TT-matrix of size + np.prod(shape[0]) x np.prod(shape[1]) + """ + # TODO: good distribution to init training. + # In case the shape is immutable. + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1]), dtype=int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0]), dtype=int) + shape = np.array(shape) + tt_rank = np.array(tt_rank) + + _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) + + num_dims = shape[0].size + if tt_rank.size == 1: + tt_rank = tt_rank * np.ones(num_dims - 1) + tt_rank = np.concatenate([[1], tt_rank, [1]]) + + tt_rank = tt_rank.astype(int) + var = np.prod(tt_rank) + + # Empirically entries of a TT tensor with cores initialized from N(0, 1) + # will have variances np.prod(tt_rank) and mean 0. + # We scale each TT-core to obtain the desired stddev + + cr_exponent = -1.0 / (2 * num_dims) + var = np.prod(tt_rank ** cr_exponent) + core_stddev = stddev ** (1.0 / num_dims) * var + tt = matrix_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) + + if np.abs(mean) < 1e-8: + return tt + else: + raise NotImplementedError('non-zero mean is not supported yet') + + +def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): + """Generate a batch of TT-matrices with given shape, mean and stddev. + + Entries of the generated matrices (in the full format) will be iid and + satisfy E[x_{i1i2..id}] = mean, Var[x_{i1i2..id}] = stddev^2, but the + distribution is in fact not Gaussian. + + In the current implementation only mean 0 is supported. To get a + random_matrix_batch with specified mean but tt_rank greater by 1 you can call + x = t3f.random_matrix_batch(shape, tt_rank, batch_size=bs, stddev=stddev) + x = mean * t3f.ones_like(x) + x + + Args: + shape: 2d array, shape[0] is the shape of the matrix row-index, + shape[1] is the shape of the column index. + shape[0] and shape[1] should have the same number of elements (d) + Also supports omitting one of the dimensions for vectors, e.g. + random_matrix_batch([[2, 2, 2], None]) + and + random_matrix_batch([None, [2, 2, 2]]) + will create a batch of one 8-element column and row vector correspondingly. + tt_rank: a number or a (d+1)-element array with ranks. + batch_size: an integer. + mean: a number, the desired mean for the distribution of entries. + stddev: a number, the desired standard deviation for the distribution of + entries. + + Returns: + TensorTrainBatch containing a batch of TT-matrices of size + np.prod(shape[0]) x np.prod(shape[1]) + """ + # In case the shape is immutable. + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1]), dtype=int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0]), dtype=int) + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank, + batch_size=batch_size) + num_dims = shape[0].size + if tt_rank.size == 1: + tt_rank = tt_rank * np.ones(num_dims - 1) + tt_rank = np.concatenate([[1], tt_rank, [1]]) + + shape = shape.astype(int) + tt_rank = tt_rank.astype(int) + + cr_exponent = -1.0 / (2 * num_dims) + var = np.prod(tt_rank ** cr_exponent) + core_stddev = stddev ** (1.0 / num_dims) * var + tt = matrix_batch_with_random_cores(shape, tt_rank=tt_rank, + stddev=core_stddev, + batch_size=batch_size) + + if np.abs(mean) < 1e-8: + return tt + else: + raise NotImplementedError('non-zero mean is not supported yet') + + +def glorot_initializer(shape, tt_rank=2): + """Constructs a random TT matrix with entrywise variance 2.0 / (n_in + n_out) + + Args: + shape: 2d array, shape[0] is the shape of the matrix row-index, + shape[1] is the shape of the column index. + shape[0] and shape[1] should have the same number of elements (d) + Also supports omitting one of the dimensions for vectors, e.g. + glorot_initializer([[2, 2, 2], None]) + and + glorot_initializer([None, [2, 2, 2]]) + will create an 8-element column and row vectors correspondingly. + tt_rank: a number or a (d+1)-element array with ranks. + + Returns: + TensorTrain containing a TT-matrix of size + np.prod(shape[0]) x np.prod(shape[1]) + """ + + # In case the shape is immutable. + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1]), dtype=int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0]), dtype=int) + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) + n_in = np.prod(shape[0]) + n_out = np.prod(shape[1]) + lamb = 2.0 / (n_in + n_out) + + return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) + + +def he_initializer(shape, tt_rank=2): + """Constructs a random TT matrix with entrywise variance 2.0 / n_in + + Args: + shape: 2d array, shape[0] is the shape of the matrix row-index, + shape[1] is the shape of the column index. + shape[0] and shape[1] should have the same number of elements (d) + Also supports omitting one of the dimensions for vectors, e.g. + he_initializer([[2, 2, 2], None]) + and + he_initializer([None, [2, 2, 2]]) + will create an 8-element column and row vectors correspondingly. + tt_rank: a number or a (d+1)-element array with ranks. + + Returns: + TensorTrain containing a TT-matrix of size + np.prod(shape[0]) x np.prod(shape[1]) + """ + + # In case the shape is immutable. + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1]), dtype=int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0]), dtype=int) + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) + n_in = np.prod(shape[0]) + lamb = 2.0 / n_in + + return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) + + +def lecun_initializer(shape, tt_rank=2): + """Constructs a random TT matrix with entrywise variance 1.0 / n_in + + Args: + shape: 2d array, shape[0] is the shape of the matrix row-index, + shape[1] is the shape of the column index. + shape[0] and shape[1] should have the same number of elements (d) + Also supports omitting one of the dimensions for vectors, e.g. + lecun_initializer([[2, 2, 2], None]) + and + lecun_initializer([None, [2, 2, 2]]) + will create an 8-element column and row vectors correspondingly. + tt_rank: a number or a (d+1)-element array with ranks. + + Returns: + TensorTrain containing a TT-matrix of size + np.prod(shape[0]) x np.prod(shape[1]) + """ + + # In case the shape is immutable. + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1]), dtype=int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0]), dtype=int) + shape = np.array(shape) + tt_rank = np.array(tt_rank) + _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) + n_in = np.prod(shape[0]) + lamb = 1.0 / n_in + return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) diff --git a/build/lib/t3f/initializers_test.py b/build/lib/t3f/initializers_test.py new file mode 100644 index 00000000..a555eeba --- /dev/null +++ b/build/lib/t3f/initializers_test.py @@ -0,0 +1,176 @@ +import numpy as np +import tensorflow as tf + +from t3f import initializers +from t3f import ops + + +class InitializersTest(tf.test.TestCase): + + def testTensorOnesAndZeros(self): + tt_ones = initializers.tensor_ones([2, 3, 4]) + tt_zeros = initializers.tensor_zeros([2, 3, 4]) + + ones_desired = np.ones((2, 3, 4)) + zeros_desired = np.zeros((2, 3, 4)) + with self.test_session() as sess: + tt_ones_full = sess.run(ops.full(tt_ones)) + tt_zeros_full = sess.run(ops.full(tt_zeros)) + self.assertAllClose(tt_ones_full, ones_desired) + self.assertAllClose(tt_zeros_full, zeros_desired) + bad_shapes = [[[2, 3]], [-1, 3], [0.1, 4]] + for shape in bad_shapes: + with self.assertRaises(ValueError): + initializers.tensor_ones(shape) + with self.assertRaises(ValueError): + initializers.tensor_zeros(shape) + + def testMatrixOnesAndZeros(self): + tt_ones = initializers.matrix_ones([[2, 3, 4], [1, 2, 5]]) + tt_zeros = initializers.matrix_zeros([[2, 3, 4], [1, 2, 5]]) + + ones_desired = np.ones((24, 10)) + zeros_desired = np.zeros((24, 10)) + + bad_shapes = [[[-1, 2, 3], [3, 4, 6]], [[1.5, 2, 4], [2, 5, 6]], + [[1], [2, 3]], [2, 3, 4]] + with self.test_session() as sess: + tt_ones_full = sess.run(ops.full(tt_ones)) + tt_zeros_full = sess.run(ops.full(tt_zeros)) + self.assertAllClose(tt_ones_full, ones_desired) + self.assertAllClose(tt_zeros_full, zeros_desired) + for shape in bad_shapes: + with self.assertRaises(ValueError): + initializers.matrix_ones(shape) + with self.assertRaises(ValueError): + initializers.matrix_zeros(shape) + + def testEye(self): + tt_eye = initializers.eye([4, 5, 6]) + eye_desired = np.eye(120) + with self.test_session() as sess: + eye_full = sess.run(ops.full(tt_eye)) + self.assertAllClose(eye_full, eye_desired) + bad_shapes = [[[2, 3]], [-1, 3], [0.1, 4]] + for shape in bad_shapes: + with self.assertRaises(ValueError): + initializers.eye(shape) + + def testOnesLikeAndZerosLike(self): + a = initializers.random_tensor([2, 3, 4]) + b = initializers.ones_like(a) + c = initializers.zeros_like(a) + var_list = [ops.full(b), ops.full(c)] + with self.test_session() as sess: + bf, cf = sess.run(var_list) + self.assertAllClose(bf, np.ones((2, 3, 4))) + self.assertAllClose(cf, np.zeros((2, 3, 4))) + with self.assertRaises(ValueError): + initializers.ones_like(1) + with self.assertRaises(ValueError): + initializers.zeros_like(1) + + def testRandomTensor(self): + shapes = [[3, 4], [3, 4], [3, 4], [3, 4], [1, -2], [1.1, 2], [[3, 4]]] + tt_ranks = [-2, 1.5, [2, 3, 4, 5], [1.5], 2, 2, 2] + bad_cases = zip(shapes, tt_ranks) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.random_tensor(case[0], tt_rank=case[1]) + + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.tensor_with_random_cores(case[0], tt_rank=case[1]) + + with self.assertRaises(NotImplementedError): + initializers.random_tensor([1, 2], mean=1.0) + + def testRandomMatrix(self): + shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], + [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]]] + tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3]] + bad_cases = zip(shapes, tt_ranks) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.random_matrix(case[0], tt_rank=case[1]) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.matrix_with_random_cores(case[0], tt_rank=case[1]) + with self.assertRaises(NotImplementedError): + initializers.random_matrix([[2, 3, 4], [1, 2, 3]], mean=1.0) + + def testRandomTensorBatch(self): + shapes = [[3, 4], [3, 4], [3, 4], [3, 4], [1, -2], [1.1, 2], [[3, 4]], + [1, 2], [3, 4]] + tt_ranks = [-2, 1.5, [2, 3, 4, 5], [1.5], 2, 2, 2, 2, 2] + bs = [1] * 7 + [-1] + [0.5] + bad_cases = zip(shapes, tt_ranks, bs) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.random_tensor_batch(case[0], tt_rank=case[1], + batch_size=case[2]) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.tensor_batch_with_random_cores(case[0], tt_rank=case[1], + batch_size=case[2]) + with self.assertRaises(NotImplementedError): + initializers.random_tensor_batch([1, 2, 3], mean=1.0) + + def testRandomMatrixBatch(self): + shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], + [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]]] + tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3], 2, 2] + bs = 7 * [1] + [-1] + [0.5] + bad_cases = zip(shapes, tt_ranks, bs) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.random_matrix_batch(case[0], tt_rank=case[1], + batch_size=case[2]) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.matrix_batch_with_random_cores(case[0], tt_rank=case[1], + batch_size=case[2]) + with self.assertRaises(NotImplementedError): + initializers.random_matrix_batch([[1, 2, 3], [1, 2, 3]], mean=1.0) + + def testGlorot(self): + shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], + [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]]] + tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3]] + bad_cases = zip(shapes, tt_ranks) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.glorot_initializer(case[0], tt_rank=case[1]) + + def testHe(self): + shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], + [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]]] + tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3]] + bad_cases = zip(shapes, tt_ranks) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.he_initializer(case[0], tt_rank=case[1]) + + def testLecun(self): + shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], + [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3]]] + tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3]] + bad_cases = zip(shapes, tt_ranks) + for case in bad_cases: + with self.assertRaises(ValueError): + initializers.lecun_initializer(case[0], tt_rank=case[1]) + + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/kronecker.py b/build/lib/t3f/kronecker.py new file mode 100644 index 00000000..0910c364 --- /dev/null +++ b/build/lib/t3f/kronecker.py @@ -0,0 +1,237 @@ +import tensorflow as tf + +from t3f.tensor_train import TensorTrain +from t3f.tensor_train_batch import TensorTrainBatch +from t3f import ops + + +def determinant(kron_a): + """Computes the determinant of a given Kronecker-factorized matrix. + + Note, that this method can suffer from overflow. + + Args: + kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a + batch of matrices of size N x N, factorized into a Kronecker product of + square matrices (all tt-ranks are 1 and all tt-cores are square). + + Returns: + A number or a Tensor with numbers for each element in the batch. + The determinant of the given matrix. + + Raises: + ValueError if the tt-cores of the provided matrix are not square, + or the tt-ranks are not 1. + """ + if not _is_kron(kron_a): + raise ValueError('The argument should be a Kronecker product (tt-ranks ' + 'should be 1)') + + shapes_defined = kron_a.get_shape().is_fully_defined() + if shapes_defined: + i_shapes = kron_a.get_raw_shape()[0] + j_shapes = kron_a.get_raw_shape()[1] + else: + i_shapes = ops.raw_shape(kron_a)[0] + j_shapes = ops.raw_shape(kron_a)[1] + + if shapes_defined: + if i_shapes != j_shapes: + raise ValueError('The argument should be a Kronecker product of square ' + 'matrices (tt-cores must be square)') + + is_batch = isinstance(kron_a, TensorTrainBatch) + pows = tf.cast(tf.reduce_prod(i_shapes), kron_a.dtype) + cores = kron_a.tt_cores + det = 1 + for core_idx in range(kron_a.ndims()): + core = cores[core_idx] + if is_batch: + core_det = tf.matrix_determinant(core[:, 0, :, :, 0]) + else: + core_det = tf.matrix_determinant(core[0, :, :, 0]) + core_pow = pows / i_shapes[core_idx].value + + det *= tf.pow(core_det, core_pow) + return det + + +def slog_determinant(kron_a): + """Computes the sign and log-det of a given Kronecker-factorized matrix. + + Args: + kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a + batch of matrices of size N x N, factorized into a Kronecker product of + square matrices (all tt-ranks are 1 and all tt-cores are square). + + Returns: + Two number or two Tensor with numbers for each element in the batch. + Sign of the determinant and the log-determinant of the given + matrix. If the determinant is zero, then sign will be 0 and logdet will be + -Inf. In all cases, the determinant is equal to sign * np.exp(logdet). + + Raises: + ValueError if the tt-cores of the provided matrix are not square, + or the tt-ranks are not 1. + """ + if not _is_kron(kron_a): + raise ValueError('The argument should be a Kronecker product ' + '(tt-ranks should be 1)') + + shapes_defined = kron_a.get_shape().is_fully_defined() + if shapes_defined: + i_shapes = kron_a.get_raw_shape()[0] + j_shapes = kron_a.get_raw_shape()[1] + else: + i_shapes = ops.raw_shape(kron_a)[0] + j_shapes = ops.raw_shape(kron_a)[1] + + if shapes_defined: + if i_shapes != j_shapes: + raise ValueError('The argument should be a Kronecker product of square ' + 'matrices (tt-cores must be square)') + + is_batch = isinstance(kron_a, TensorTrainBatch) + pows = tf.cast(tf.reduce_prod(i_shapes), kron_a.dtype) + logdet = 0. + det_sign = 1. + + for core_idx in range(kron_a.ndims()): + core = kron_a.tt_cores[core_idx] + if is_batch: + core_det = tf.matrix_determinant(core[:, 0, :, :, 0]) + else: + core_det = tf.matrix_determinant(core[0, :, :, 0]) + core_abs_det = tf.abs(core_det) + core_det_sign = tf.sign(core_det) + core_pow = pows / i_shapes[core_idx].value + logdet += tf.log(core_abs_det) * core_pow + det_sign *= core_det_sign**(core_pow) + return det_sign, logdet + + +def inv(kron_a): + """Computes the inverse of a given Kronecker-factorized matrix. + + Args: + kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a + batch of matrices of size N x N, factorized into a Kronecker product of + square matrices (all tt-ranks are 1 and all tt-cores are square). + + Returns: + `TensorTrain` object containing a TT-matrix of size N x N if the argument is + `TensorTrain` + `TensorTrainBatch` object, containing TT-matrices of size N x N if the + argument is `TensorTrainBatch` + + Raises: + ValueError if the tt-cores of the provided matrix are not square, + or the tt-ranks are not 1. + """ + if not _is_kron(kron_a): + raise ValueError('The argument should be a Kronecker product ' + '(tt-ranks should be 1)') + + shapes_defined = kron_a.get_shape().is_fully_defined() + if shapes_defined: + i_shapes = kron_a.get_raw_shape()[0] + j_shapes = kron_a.get_raw_shape()[1] + else: + i_shapes = ops.raw_shape(kron_a)[0] + j_shapes = ops.raw_shape(kron_a)[1] + + if shapes_defined: + if i_shapes != j_shapes: + raise ValueError('The argument should be a Kronecker product of square ' + 'matrices (tt-cores must be square)') + + is_batch = isinstance(kron_a, TensorTrainBatch) + inv_cores = [] + for core_idx in range(kron_a.ndims()): + core = kron_a.tt_cores[core_idx] + if is_batch: + core_inv = tf.matrix_inverse(core[:, 0, :, :, 0]) + core_inv = tf.expand_dims(tf.expand_dims(core_inv, 1), -1) + else: + core_inv = tf.matrix_inverse(core[0, :, :, 0]) + core_inv = tf.expand_dims(tf.expand_dims(core_inv, 0), -1) + inv_cores.append(core_inv) + + res_ranks = kron_a.get_tt_ranks() + res_shape = kron_a.get_raw_shape() + if is_batch: + return TensorTrainBatch(inv_cores, res_shape, res_ranks) + else: + return TensorTrain(inv_cores, res_shape, res_ranks) + + +def cholesky(kron_a): + """Computes the Cholesky decomposition of a given Kronecker-factorized matrix. + + Args: + kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a + batch of matrices of size N x N, factorized into a Kronecker product of + square matrices (all tt-ranks are 1 and all tt-cores are square). All the + cores must be symmetric positive-definite. + + Returns: + `TensorTrain` object containing a TT-matrix of size N x N if the argument is + `TensorTrain` + `TensorTrainBatch` object, containing TT-matrices of size N x N if the + argument is `TensorTrainBatch` + + Raises: + ValueError if the tt-cores of the provided matrix are not square, + or the tt-ranks are not 1. + """ + if not _is_kron(kron_a): + raise ValueError('The argument should be a Kronecker product ' + '(tt-ranks should be 1)') + + shapes_defined = kron_a.get_shape().is_fully_defined() + if shapes_defined: + i_shapes = kron_a.get_raw_shape()[0] + j_shapes = kron_a.get_raw_shape()[1] + else: + i_shapes = ops.raw_shape(kron_a)[0] + j_shapes = ops.raw_shape(kron_a)[1] + + if shapes_defined: + if i_shapes != j_shapes: + raise ValueError('The argument should be a Kronecker product of square ' + 'matrices (tt-cores must be square)') + + is_batch = isinstance(kron_a, TensorTrainBatch) + cho_cores = [] + + for core_idx in range(kron_a.ndims()): + core = kron_a.tt_cores[core_idx] + if is_batch: + core_cho = tf.cholesky(core[:, 0, :, :, 0]) + core_cho = tf.expand_dims(tf.expand_dims(core_cho, 1), -1) + else: + core_cho = tf.cholesky(core[0, :, :, 0]) + core_cho = tf.expand_dims(tf.expand_dims(core_cho, 0), -1) + cho_cores.append(core_cho) + + res_ranks = kron_a.get_tt_ranks() + res_shape = kron_a.get_raw_shape() + if is_batch: + return TensorTrainBatch(cho_cores, res_shape, res_ranks) + else: + return TensorTrain(cho_cores, res_shape, res_ranks) + + +def _is_kron(tt_a): + """Returns True if the argument is a Kronecker product matrix. + + Args: + t_a: `TensorTrain` or `TensorTrainBatch` object. + + Returns: + bool + """ + if tt_a.is_tt_matrix(): + return max(tt_a.get_tt_ranks()) == 1 + return False + diff --git a/build/lib/t3f/kronecker_test.py b/build/lib/t3f/kronecker_test.py new file mode 100644 index 00000000..84a07940 --- /dev/null +++ b/build/lib/t3f/kronecker_test.py @@ -0,0 +1,182 @@ +import numpy as np +import tensorflow as tf + +from t3f.tensor_train import TensorTrain +from t3f.tensor_train_batch import TensorTrainBatch +from t3f import ops +from t3f import initializers +from t3f import variables +from t3f import kronecker as kr + +class KroneckerTest(tf.test.TestCase): + + def testIsKronNonKron(self): + # Tests _is_kron on a non-Kronecker matrix + initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=2) + tt_mat = variables.get_variable('tt_mat', initializer=initializer) + self.assertFalse(kr._is_kron(tt_mat)) + + def testIsKronKron(self): + # Tests _is_kron on a Kronecker matrix + initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=1) + kron_mat = variables.get_variable('kron_mat', initializer=initializer) + self.assertTrue(kr._is_kron(kron_mat)) + + def testDet(self): + # Tests the determinant function + initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1) + kron_mat = variables.get_variable('kron_mat', initializer=initializer) + init_op = tf.global_variables_initializer() + with self.test_session() as sess: + sess.run(init_op) + desired = np.linalg.det(ops.full(kron_mat).eval()) + actual = kr.determinant(kron_mat).eval() + self.assertAllClose(desired, actual) + + def testSlogDet(self): + # Tests the slog_determinant function + + # TODO: use kron and -1 * kron matrices, when mul is implemented + # the current version is platform-dependent + + tf.set_random_seed(5) # negative derminant + initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1) + kron_neg = variables.get_variable('kron_neg', initializer=initializer) + + tf.set_random_seed(1) # positive determinant + initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1) + kron_pos = variables.get_variable('kron_pos', initializer=initializer) + + init_op = tf.global_variables_initializer() + with self.test_session() as sess: + # negative derminant + sess.run(init_op) + desired_sign, desired_det = np.linalg.slogdet(ops.full(kron_neg).eval()) + actual_sign, actual_det = sess.run(kr.slog_determinant(kron_neg)) + self.assertEqual(desired_sign, actual_sign) + self.assertAllClose(desired_det, actual_det) + + # positive determinant + desired_sign, desired_det = np.linalg.slogdet(ops.full(kron_pos).eval()) + actual_sign, actual_det = sess.run(kr.slog_determinant(kron_pos)) + self.assertEqual(desired_sign, actual_sign) + self.assertAllClose(desired_det, actual_det) + + def testInv(self): + # Tests the inv function + initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1) + kron_mat = variables.get_variable('kron_mat', initializer=initializer) + init_op = tf.global_variables_initializer() + with self.test_session() as sess: + sess.run(init_op) + desired = np.linalg.inv(ops.full(kron_mat).eval()) + actual = ops.full(kr.inv(kron_mat)).eval() + self.assertAllClose(desired, actual) + + def testCholesky(self): + # Tests the cholesky function + np.random.seed(8) + + # generating two symmetric positive-definite tt-cores + L_1 = np.tril(np.random.normal(scale=2., size=(2, 2))) + L_2 = np.tril(np.random.normal(scale=2., size=(3, 3))) + K_1 = L_1.dot(L_1.T) + K_2 = L_2.dot(L_2.T) + K = np.kron(K_1, K_2) + initializer = TensorTrain([K_1[None, :, :, None], + K_2[None, :, :, None]], + tt_ranks=7*[1]) + kron_mat = variables.get_variable('kron_mat', initializer=initializer) + init_op = tf.global_variables_initializer() + with self.test_session() as sess: + sess.run(init_op) + desired = np.linalg.cholesky(K) + actual = ops.full(kr.cholesky(kron_mat)).eval() + self.assertAllClose(desired, actual) + +class BatchKroneckerTest(tf.test.TestCase): + + def testIsKronNonKron(self): + # Tests _is_kron on a non-Kronecker matrix batch + initializer = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=2, + batch_size=3) + tt_mat_batch = variables.get_variable('tt_mat_batch', + initializer=initializer) + self.assertFalse(kr._is_kron(tt_mat_batch)) + + def testIsKronKron(self): + # Tests _is_kron on a Kronecker matrix batch + initializer = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=1, + batch_size=3) + kron_mat_batch = variables.get_variable('kron_mat_batch', + initializer=initializer) + self.assertTrue(kr._is_kron(kron_mat_batch)) + + def testDet(self): + # Tests the determinant function + initializer = initializers.random_matrix_batch(((2, 3, 2), (2, 3, 2)), + tt_rank=1, batch_size=3) + kron_mat_batch = variables.get_variable('kron_mat_batch', + initializer=initializer) + init_op = tf.global_variables_initializer() + with self.test_session() as sess: + sess.run(init_op) + desired = tf.matrix_determinant(ops.full(kron_mat_batch)).eval() + actual = kr.determinant(kron_mat_batch).eval() + self.assertAllClose(desired, actual) + + def testSlogDet(self): + # Tests the slog_determinant function + + tf.set_random_seed(1) # negative and positive determinants + initializer = initializers.random_matrix_batch(((2, 3), (2, 3)), tt_rank=1, + batch_size=3) + kron_mat_batch = variables.get_variable('kron_mat_batch', + initializer=initializer) + + init_op = tf.global_variables_initializer() + with self.test_session() as sess: + # negative derminant + sess.run(init_op) + desired_sign, desired_det = np.linalg.slogdet( + ops.full(kron_mat_batch).eval()) + actual_sign, actual_det = sess.run(kr.slog_determinant(kron_mat_batch)) + self.assertAllEqual(desired_sign, actual_sign) + self.assertAllClose(desired_det, actual_det) + + def testInv(self): + # Tests the inv function + initializer = initializers.random_matrix_batch(((2, 3, 2), (2, 3, 2)), + tt_rank=1, batch_size=3) + kron_mat_batch = variables.get_variable('kron_mat_batch', + initializer=initializer) + init_op = tf.global_variables_initializer() + with self.test_session() as sess: + sess.run(init_op) + desired = np.linalg.inv(ops.full(kron_mat_batch).eval()) + actual = ops.full(kr.inv(kron_mat_batch)).eval() + self.assertAllClose(desired, actual, atol=1e-4) + + def testCholesky(self): + # Tests the cholesky function + np.random.seed(8) + + # generating two symmetric positive-definite tt-cores + L_1 = np.tril(np.random.normal(scale=2., size=(4, 2, 2))) + L_2 = np.tril(np.random.normal(scale=2., size=(4, 3, 3))) + K_1 = np.einsum('ijk,ilk->ijl', L_1, L_1) + K_2 = np.einsum('ijk,ilk->ijl', L_2, L_2) + initializer = TensorTrainBatch([K_1[:, None, :, :, None], + K_2[:, None, :, :, None]], tt_ranks=7*[1]) + kron_mat_batch = variables.get_variable('kron_mat_batch', + initializer=initializer) + init_op = tf.global_variables_initializer() + with self.test_session() as sess: + sess.run(init_op) + desired = np.linalg.cholesky(ops.full(kron_mat_batch).eval()) + actual = ops.full(kr.cholesky(kron_mat_batch)).eval() + self.assertAllClose(desired, actual) + + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/ops.py b/build/lib/t3f/ops.py new file mode 100644 index 00000000..0ea892db --- /dev/null +++ b/build/lib/t3f/ops.py @@ -0,0 +1,1204 @@ +import tensorflow as tf +import numpy as np +from t3f.tensor_train_base import TensorTrainBase +from t3f.tensor_train import TensorTrain +from t3f.tensor_train_batch import TensorTrainBatch +from t3f import shapes +from t3f import utils +from t3f import decompositions +from t3f import initializers + +# TODO: add complexities to the comments. + + +def full(tt): + """Converts a TensorTrain into a regular tensor or matrix (tf.Tensor). + + Args: + tt: `TensorTrain` or `TensorTrainBatch` object. + + Returns: + tf.Tensor. + """ + if isinstance(tt, TensorTrainBatch): + # Batch of Tensor Trains. + return _full_tt_batch(tt) + else: + # TensorTrain object (not batch). + return _full_tt(tt) + + +def _full_tt(tt): + """Converts a TensorTrain into a regular tensor or matrix (tf.Tensor). + + Args: + tt: `TensorTrain` object. + + Returns: + tf.Tensor. + """ + num_dims = tt.ndims() + ranks = shapes.lazy_tt_ranks(tt) + shape = shapes.lazy_shape(tt) + raw_shape = shapes.lazy_raw_shape(tt) + + res = tt.tt_cores[0] + for i in range(1, num_dims): + res = tf.reshape(res, (-1, ranks[i])) + curr_core = tf.reshape(tt.tt_cores[i], (ranks[i], -1)) + res = tf.matmul(res, curr_core) + if tt.is_tt_matrix(): + intermediate_shape = [] + for i in range(num_dims): + intermediate_shape.append(raw_shape[0][i]) + intermediate_shape.append(raw_shape[1][i]) + res = tf.reshape(res, intermediate_shape) + transpose = [] + for i in range(0, 2 * num_dims, 2): + transpose.append(i) + for i in range(1, 2 * num_dims, 2): + transpose.append(i) + res = tf.transpose(res, transpose) + return tf.reshape(res, shape) + else: + return tf.reshape(res, shape) + + +def _full_tt_batch(tt): + """Converts a TensorTrainBatch into a regular tensor or matrix (tf.Tensor). + + Args: + tt: `TensorTrainBatch` object. + + Returns: + tf.Tensor. + """ + num_dims = tt.ndims() + ranks = shapes.lazy_tt_ranks(tt) + shape = shapes.lazy_shape(tt) + raw_shape = shapes.lazy_raw_shape(tt) + + res = tt.tt_cores[0] + batch_size = shapes.lazy_batch_size(tt) + for i in range(1, num_dims): + res = tf.reshape(res, (batch_size, -1, ranks[i])) + curr_core = tf.reshape(tt.tt_cores[i], (batch_size, ranks[i], -1)) + res = tf.einsum('oqb,obw->oqw', res, curr_core) + if tt.is_tt_matrix(): + intermediate_shape = [batch_size] + for i in range(num_dims): + intermediate_shape.append(raw_shape[0][i]) + intermediate_shape.append(raw_shape[1][i]) + res = tf.reshape(res, intermediate_shape) + transpose = [0] + for i in range(0, 2 * num_dims, 2): + transpose.append(i + 1) + for i in range(1, 2 * num_dims, 2): + transpose.append(i + 1) + res = tf.transpose(res, transpose) + return tf.reshape(res, shape) + else: + return tf.reshape(res, shape) + + +def tt_tt_matmul(tt_matrix_a, tt_matrix_b): + """Multiplies two TT-matrices and returns the TT-matrix of the result. + + Args: + tt_matrix_a: `TensorTrain` or `TensorTrainBatch` object containing + a TT-matrix (a batch of TT-matrices) of size M x N + tt_matrix_b: `TensorTrain` or `TensorTrainBatch` object containing + a TT-matrix (a batch of TT-matrices) of size N x P + + Returns + `TensorTrain` object containing a TT-matrix of size M x P if both arguments + are `TensorTrain`s + `TensorTrainBatch` if any of the arguments is a `TensorTrainBatch` + + Raises: + ValueError is the arguments are not TT matrices or if their sizes are not + appropriate for a matrix-by-matrix multiplication. + """ + # Both TensorTrain and TensorTrainBatch are inherited from TensorTrainBase. + if not isinstance(tt_matrix_a, TensorTrainBase) or \ + not isinstance(tt_matrix_b, TensorTrainBase) or \ + not tt_matrix_a.is_tt_matrix() or \ + not tt_matrix_b.is_tt_matrix(): + raise ValueError('Arguments should be TT-matrices') + + if not shapes.is_batch_broadcasting_possible(tt_matrix_a, tt_matrix_b): + raise ValueError('The batch sizes are different and not 1, broadcasting is ' + 'not available.') + + ndims = tt_matrix_a.ndims() + if tt_matrix_b.ndims() != ndims: + raise ValueError('Arguments should have the same number of dimensions, ' + 'got %d and %d instead.' % (ndims, tt_matrix_b.ndims())) + + # Convert BatchSize 1 batch into TT object to simplify broadcasting. + tt_matrix_a = shapes.squeeze_batch_dim(tt_matrix_a) + tt_matrix_b = shapes.squeeze_batch_dim(tt_matrix_b) + is_a_batch = isinstance(tt_matrix_a, TensorTrainBatch) + is_b_batch = isinstance(tt_matrix_b, TensorTrainBatch) + is_res_batch = is_a_batch or is_b_batch + a_batch_str = 'o' if is_a_batch else '' + b_batch_str = 'o' if is_b_batch else '' + res_batch_str = 'o' if is_res_batch else '' + einsum_str = '{}aijb,{}cjkd->{}acikbd'.format(a_batch_str, b_batch_str, + res_batch_str) + result_cores = [] + # TODO: name the operation and the resulting tensor. + a_shape = shapes.lazy_raw_shape(tt_matrix_a) + a_ranks = shapes.lazy_tt_ranks(tt_matrix_a) + b_shape = shapes.lazy_raw_shape(tt_matrix_b) + b_ranks = shapes.lazy_tt_ranks(tt_matrix_b) + if is_res_batch: + if is_a_batch: + batch_size = shapes.lazy_batch_size(tt_matrix_a) + if is_b_batch: + batch_size = shapes.lazy_batch_size(tt_matrix_b) + for core_idx in range(ndims): + a_core = tt_matrix_a.tt_cores[core_idx] + b_core = tt_matrix_b.tt_cores[core_idx] + curr_res_core = tf.einsum(einsum_str, a_core, b_core) + + res_left_rank = a_ranks[core_idx] * b_ranks[core_idx] + res_right_rank = a_ranks[core_idx + 1] * b_ranks[core_idx + 1] + left_mode = a_shape[0][core_idx] + right_mode = b_shape[1][core_idx] + if is_res_batch: + core_shape = (batch_size, res_left_rank, left_mode, right_mode, res_right_rank) + else: + core_shape = (res_left_rank, left_mode, right_mode, + res_right_rank) + curr_res_core = tf.reshape(curr_res_core, core_shape) + result_cores.append(curr_res_core) + + res_shape = (tt_matrix_a.get_raw_shape()[0], tt_matrix_b.get_raw_shape()[1]) + static_a_ranks = tt_matrix_a.get_tt_ranks() + static_b_ranks = tt_matrix_b.get_tt_ranks() + out_ranks = [a_r * b_r for a_r, b_r in zip(static_a_ranks, static_b_ranks)] + if is_res_batch: + return TensorTrainBatch(result_cores, res_shape, out_ranks, batch_size) + else: + return TensorTrain(result_cores, res_shape, out_ranks) + + +def tt_dense_matmul(tt_matrix_a, matrix_b): + """Multiplies a TT-matrix by a regular matrix, returns a regular matrix. + + Args: + tt_matrix_a: `TensorTrain` object containing a TT-matrix of size M x N + matrix_b: tf.Tensor of size N x P + + Returns + tf.Tensor of size M x P + """ + if not isinstance(tt_matrix_a, TensorTrain) or not tt_matrix_a.is_tt_matrix(): + raise ValueError('The first argument should be a TT-matrix') + + ndims = tt_matrix_a.ndims() + a_columns = tt_matrix_a.get_shape()[1].value + b_rows = matrix_b.get_shape()[0].value + if a_columns is not None and b_rows is not None: + if a_columns != b_rows: + raise ValueError('Arguments shapes should align got %d and %d instead.' % + (tt_matrix_a.get_shape(), matrix_b.get_shape())) + + a_shape = shapes.lazy_shape(tt_matrix_a) + a_raw_shape = shapes.lazy_raw_shape(tt_matrix_a) + if matrix_b.get_shape().is_fully_defined(): + b_shape = matrix_b.get_shape().as_list() + else: + b_shape = tf.shape(matrix_b) + a_ranks = shapes.lazy_tt_ranks(tt_matrix_a) + # If A is (i0, ..., id-1) x (j0, ..., jd-1) and B is (j0, ..., jd-1) x K, + # data is (K, j0, ..., jd-2) x jd-1 x 1 + data = tf.transpose(matrix_b) + data = tf.reshape(data, (-1, a_raw_shape[1][-1], 1)) + for core_idx in reversed(range(ndims)): + curr_core = tt_matrix_a.tt_cores[core_idx] + # On the k = core_idx iteration, after applying einsum the shape of data + # becomes ik x (ik-1..., id-1, K, j0, ..., jk-1) x rank_k + data = tf.einsum('aijb,rjb->ira', curr_core, data) + if core_idx > 0: + # After reshape the shape of data becomes + # (ik, ..., id-1, K, j0, ..., jk-2) x jk-1 x rank_k + new_data_shape = (-1, a_raw_shape[1][core_idx - 1], a_ranks[core_idx]) + data = tf.reshape(data, new_data_shape) + # At the end the shape of the data is (i0, ..., id-1) x K + return tf.reshape(data, (a_shape[0], b_shape[1])) + + +def dense_tt_matmul(matrix_a, tt_matrix_b): + """Multiplies a regular matrix by a TT-matrix, returns a regular matrix. + + Args: + matrix_a: tf.Tensor of size M x N + tt_matrix_b: `TensorTrain` object containing a TT-matrix of size N x P + + Returns + tf.Tensor of size M x P + """ +# TODO: make a more efficient implementation. + a_t = tf.transpose(matrix_a) + b_t = transpose(tt_matrix_b) + return tf.transpose(tt_dense_matmul(b_t, a_t)) + + +def sparse_tt_matmul(sparse_matrix_a, tt_matrix_b): + """Multiplies a sparse matrix by a TT-matrix, returns a regular matrix. + + Args: + sparse_matrix_a: tf.SparseTensor of size M x N + tt_matrix_b: `TensorTrain` object containing a TT-matrix of size N x P + + Returns + tf.Tensor of size M x P + """ + raise NotImplementedError + + +# TODO: add flag `return_type = (TT | dense)`? +def tt_sparse_matmul(tt_matrix_a, sparse_matrix_b): + """Multiplies a TT-matrix by a sparse matrix, returns a regular matrix. + + Args: + tt_matrix_a: `TensorTrain` object containing a TT-matrix of size M x N + sparse_matrix_b: tf.SparseTensor of size N x P + + Returns + tf.Tensor of size M x P + """ + raise NotImplementedError + + +def matmul(a, b): + """Multiplies two matrices that can be TT-, dense, or sparse. + + Note that multiplication of two TT-matrices returns a TT-matrix with much + larger ranks. + Also works for multiplying two batches of TT-matrices or a product between a + TT-matrix and a batch of TT-matrices (with broadcasting). + + Args: + a: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor of + size M x N + b: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor of + size N x P + + Returns + If both arguments are `TensorTrain` objects, returns a `TensorTrain` + object containing a TT-matrix of size M x P. + If at least one of the arguments is a `TensorTrainBatch` object, returns + a `TensorTrainBatch` object containing a batch of TT-matrices of size + M x P. + Otherwise, returns tf.Tensor of size M x P. + """ +# TODO: is it safe to check types? What if a class is derived from TT? + if isinstance(a, TensorTrainBase) and isinstance(b, TensorTrainBase): + return tt_tt_matmul(a, b) + elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): + return tt_dense_matmul(a, b) + elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): + return dense_tt_matmul(a, b) + elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): + return tt_sparse_matmul(a, b) + elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): + return sparse_tt_matmul(a, b) + else: + raise ValueError('Argument types are not supported in matmul: %s x %s' % + (a, b)) + + +def tt_tt_flat_inner(tt_a, tt_b): + """Inner product between two TT-tensors or TT-matrices along all axis. + + The shapes of tt_a and tt_b should coincide. + + Args: + tt_a: `TensorTrain` or `TensorTrainBatch` object + tt_b: `TensorTrain` or `TensorTrainBatch` object + + Returns + a number or a Tensor with numbers for each element in the batch. + sum of products of all the elements of tt_a and tt_b + + Raises: + ValueError if the arguments are not `TensorTrain` objects, have different + number of TT-cores, different underlying shape, or if you are trying to + compute inner product between a TT-matrix and a TT-tensor. + + Complexity: + Multiplying two single TT-objects is O(d r^3 n) where d is the number of + TT-cores (tt_a.ndims()), r is the largest TT-rank + max(tt_a.get_tt_rank(), tt_b.get_tt_rank()) + and n is the size of the axis dimension, e.g. + for a tensor of size 4 x 4 x 4, n is 4; + for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 + A more precise complexity is O(d r1 r2 n max(r1, r2)) where + r1 is the largest TT-rank of tt_a and r2 is the largest TT-rank of tt_b. + The complexity of this operation for batch input is O(batch_size d r^3 n). + """ + if not isinstance(tt_a, TensorTrainBase) or not isinstance(tt_b, + TensorTrainBase): + raise ValueError('Arguments should be TensorTrains') + + if tt_a.is_tt_matrix() != tt_b.is_tt_matrix(): + raise ValueError('One of the arguments is a TT-tensor, the other is ' + 'a TT-matrix, disallowed') + are_both_matrices = tt_a.is_tt_matrix() and tt_b.is_tt_matrix() + + if not shapes.is_batch_broadcasting_possible(tt_a, tt_b): + raise ValueError('The batch sizes are different and not 1, broadcasting is ' + 'not available.') + + # TODO: compare shapes and raise if not consistent. + + ndims = tt_a.ndims() + if tt_b.ndims() != ndims: + raise ValueError('Arguments should have the same number of dimensions, ' + 'got %d and %d instead.' % (ndims, tt_b.ndims())) + + axes_str = 'ij' if are_both_matrices else 'i' + # Convert BatchSize 1 batch into TT object to simplify broadcasting. + tt_a = shapes.squeeze_batch_dim(tt_a) + tt_b = shapes.squeeze_batch_dim(tt_b) + is_a_batch = isinstance(tt_a, TensorTrainBatch) + is_b_batch = isinstance(tt_b, TensorTrainBatch) + is_res_batch = is_a_batch or is_b_batch + a_batch_str = 'o' if is_a_batch else '' + b_batch_str = 'o' if is_b_batch else '' + res_batch_str = 'o' if is_res_batch else '' + init_einsum_str = '{1}a{0}b,{2}c{0}d->{3}bd'.format(axes_str, a_batch_str, + b_batch_str, + res_batch_str) + a_core = tt_a.tt_cores[0] + b_core = tt_b.tt_cores[0] + # Simplest example of this operation: + # if both arguments are TT-tensors, then it is + # res = tf.einsum('aib,cid->bd', a_core, b_core) + res = tf.einsum(init_einsum_str, a_core, b_core) + # TODO: name the operation and the resulting tensor. + + einsum_str = '{3}ac,{1}a{0}b,{2}c{0}d->{3}bd'.format(axes_str, a_batch_str, + b_batch_str, + res_batch_str) + for core_idx in range(1, ndims): + a_core = tt_a.tt_cores[core_idx] + b_core = tt_b.tt_cores[core_idx] + # Simplest example of this operation: + # if both arguments are TT-tensors, then it is + # res = tf.einsum('ac,aib,cid->bd', res, a_core, b_core) + res = tf.einsum(einsum_str, res, a_core, b_core) + return tf.squeeze(res) + + +def tt_dense_flat_inner(tt_a, dense_b): + """Inner product between a TT-tensor (or TT-matrix) and tf.Tensor along all axis. + + The shapes of tt_a and dense_b should coincide. + + Args: + tt_a: `TensorTrain` object + dense_b: tf.Tensor + + Returns + a number + sum of products of all the elements of tt_a and dense_b + """ + raise NotImplementedError + + +def tt_sparse_flat_inner(tt_a, sparse_b): + """Inner product between a TT-tensor (or TT-matrix) and tf.SparseTensor along all axis. + + The shapes of tt_a and sparse_b should coincide. + + Args: + tt_a: `TensorTrain` object + sparse_b: tf.SparseTensor + + Returns + a number + sum of products of all the elements of tt_a and sparse_b + """ + if sparse_b.indices.get_shape().is_fully_defined(): + num_elements = sparse_b.indices.get_shape()[0] + else: + num_elements = tf.shape(sparse_b.indices)[0] + a_shape = shapes.lazy_raw_shape(tt_a) + a_ranks = shapes.lazy_tt_ranks(tt_a) + if tt_a.is_tt_matrix(): + tt_a_elements = tf.ones((num_elements, 1, 1)) + # TODO: use t3f.shape is safer?? + tensor_shape = tt_a.get_raw_shape() + row_idx_linear = tf.cast(sparse_b.indices[:, 0], tf.int64) + row_idx = utils.unravel_index(row_idx_linear, tf.cast(tensor_shape[0], tf.int64)) + col_idx_linear = tf.cast(sparse_b.indices[:, 1], tf.int64) + col_idx = utils.unravel_index(col_idx_linear, tf.cast(tensor_shape[1], tf.int64)) + for core_idx in range(tt_a.ndims()): + curr_core = tt_a.tt_cores[core_idx] + left_rank = a_ranks[core_idx] + right_rank = a_ranks[core_idx + 1] + curr_core = tf.transpose(curr_core, (1, 2, 0, 3)) + curr_core_shape = (a_shape[0][core_idx]*a_shape[1][core_idx], left_rank, + right_rank) + curr_core = tf.reshape(curr_core, curr_core_shape) + # Ravel multiindex (row_idx[:, core_idx], col_idx[:, core_idx]) into + # a linear index to use tf.gather that supports only first dimensional + # gather. + # TODO: use gather_nd instead. + curr_elements_idx = row_idx[:, core_idx] * tensor_shape[1][core_idx] + curr_elements_idx += col_idx[:, core_idx] + core_slices = tf.gather(curr_core, curr_elements_idx) + tt_a_elements = tf.matmul(tt_a_elements, core_slices) + else: + tt_a_elements = gather_nd(tt_a, sparse_b.indices) + tt_a_elements = tf.reshape(tt_a_elements, (1, -1)) + sparse_b_elements = tf.reshape(sparse_b.values, (-1, 1)) + result = tf.matmul(tt_a_elements, sparse_b_elements) + # Convert a 1x1 matrix into a number. + result = result[0, 0] + return result + + +def dense_tt_flat_inner(dense_a, tt_b): + """Inner product between a tf.Tensor and TT-tensor (or TT-matrix) along all axis. + + The shapes of dense_a and tt_b should coincide. + + Args: + dense_a: tf.Tensor + tt_b: `TensorTrain` object + + Returns + a number + sum of products of all the elements of dense_a and tt_b + """ + raise NotImplementedError + + +def sparse_tt_flat_inner(sparse_a, tt_b): + """Inner product between a tf.SparseTensor and TT-tensor (or TT-matrix) along all axis. + + The shapes of sparse_a and tt_b should coincide. + + Args: + sparse_a: tf.SparseTensor + tt_b: `TensorTrain` object + + Returns + a number + sum of products of all the elements of sparse_a and tt_b + """ + raise NotImplementedError + + +def flat_inner(a, b): + """Inner product along all axis. + + The shapes of a and b should coincide. + + Args: + a: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor + b: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor + + Returns + a number + sum of products of all the elements of a and b + OR or a tf.Tensor of size batch_size + sum of products of all the elements of a and b for each element in the + batch. + """ +# TODO: is it safe to check types? What if a class is derived from TT? + if isinstance(a, TensorTrainBase) and isinstance(b, TensorTrainBase): + return tt_tt_flat_inner(a, b) + elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): + return tt_dense_flat_inner(a, b) + elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): + return dense_tt_flat_inner(a, b) + elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): + return tt_sparse_flat_inner(a, b) + elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): + return sparse_tt_flat_inner(a, b) + else: + raise ValueError('Argument types are not supported in flat_inner: %s x %s' % + (a, b)) + + +def _add_tensor_cores(tt_a, tt_b): + """Internal function to be called from add for two TT-tensors. + + Does the actual assembling of the TT-cores to add two TT-tensors. + """ + ndims = tt_a.ndims() + dtype = tt_a.dtype + shape = shapes.lazy_raw_shape(tt_a) + a_ranks = shapes.lazy_tt_ranks(tt_a) + b_ranks = shapes.lazy_tt_ranks(tt_b) + tt_cores = [] + for core_idx in range(ndims): + a_core = tt_a.tt_cores[core_idx] + b_core = tt_b.tt_cores[core_idx] + if core_idx == 0: + curr_core = tf.concat((a_core, b_core), axis=2) + elif core_idx == ndims - 1: + curr_core = tf.concat((a_core, b_core), axis=0) + else: + upper_zeros = tf.zeros((a_ranks[core_idx], shape[0][core_idx], + b_ranks[core_idx + 1]), dtype) + lower_zeros = tf.zeros((b_ranks[core_idx], shape[0][core_idx], + a_ranks[core_idx + 1]), dtype) + upper = tf.concat((a_core, upper_zeros), axis=2) + lower = tf.concat((lower_zeros, b_core), axis=2) + curr_core = tf.concat((upper, lower), axis=0) + tt_cores.append(curr_core) + return tt_cores + + +def _add_batch_tensor_cores(tt_a, tt_b): + """Internal function to be called from add for two batches of TT-tensors. + + Does the actual assembling of the TT-cores to add two batches of TT-tensors. + """ + ndims = tt_a.ndims() + dtype = tt_a.dtype + shape = shapes.lazy_raw_shape(tt_a) + a_ranks = shapes.lazy_tt_ranks(tt_a) + b_ranks = shapes.lazy_tt_ranks(tt_b) + if isinstance(tt_a, TensorTrainBatch) and tt_a.batch_size == 1: + # We add 1 element batch tt_a to a batch_size element batch tt_b to get + # the answer TensorTrainBatch of batch_size == tt_b.batch_size. + batch_size = shapes.lazy_batch_size(tt_b) + else: + batch_size = shapes.lazy_batch_size(tt_a) + tt_a = shapes.expand_batch_dim(tt_a) + tt_b = shapes.expand_batch_dim(tt_b) + tt_cores = [] + for core_idx in range(ndims): + a_core = tt_a.tt_cores[core_idx] + if tt_a.batch_size == 1: + a_core = tf.tile(a_core, (batch_size, 1, 1, 1)) + b_core = tt_b.tt_cores[core_idx] + if tt_b.batch_size == 1: + b_core = tf.tile(b_core, (batch_size, 1, 1, 1)) + if core_idx == 0: + curr_core = tf.concat((a_core, b_core), axis=3) + elif core_idx == ndims - 1: + curr_core = tf.concat((a_core, b_core), axis=1) + else: + upper_zeros = tf.zeros((batch_size, a_ranks[core_idx], shape[0][core_idx], + b_ranks[core_idx + 1]), dtype) + lower_zeros = tf.zeros((batch_size, b_ranks[core_idx], shape[0][core_idx], + a_ranks[core_idx + 1]), dtype) + upper = tf.concat((a_core, upper_zeros), axis=3) + lower = tf.concat((lower_zeros, b_core), axis=3) + curr_core = tf.concat((upper, lower), axis=1) + tt_cores.append(curr_core) + return tt_cores, batch_size + + +def _add_matrix_cores(tt_a, tt_b): + """Internal function to be called from add for two TT-matrices. + + Does the actual assembling of the TT-cores to add two TT-matrices. + """ + ndims = tt_a.ndims() + dtype = tt_a.dtype + shape = shapes.lazy_raw_shape(tt_a) + a_ranks = shapes.lazy_tt_ranks(tt_a) + b_ranks = shapes.lazy_tt_ranks(tt_b) + tt_cores = [] + for core_idx in range(ndims): + a_core = tt_a.tt_cores[core_idx] + b_core = tt_b.tt_cores[core_idx] + if core_idx == 0: + curr_core = tf.concat((a_core, b_core), axis=3) + elif core_idx == ndims - 1: + curr_core = tf.concat((a_core, b_core), axis=0) + else: + upper_zeros = tf.zeros((a_ranks[core_idx], shape[0][core_idx], + shape[1][core_idx], b_ranks[core_idx + 1]), dtype) + lower_zeros = tf.zeros((b_ranks[core_idx], shape[0][core_idx], + shape[1][core_idx], a_ranks[core_idx + 1]), dtype) + upper = tf.concat((a_core, upper_zeros), axis=3) + lower = tf.concat((lower_zeros, b_core), axis=3) + curr_core = tf.concat((upper, lower), axis=0) + tt_cores.append(curr_core) + return tt_cores + + +def _add_batch_matrix_cores(tt_a, tt_b): + """Internal function to be called from add for two batches of TT-matrices. + + Does the actual assembling of the TT-cores to add two batches of TT-matrices. + """ + ndims = tt_a.ndims() + dtype = tt_a.dtype + shape = shapes.lazy_raw_shape(tt_a) + a_ranks = shapes.lazy_tt_ranks(tt_a) + b_ranks = shapes.lazy_tt_ranks(tt_b) + if isinstance(tt_a, TensorTrainBatch) and tt_a.batch_size == 1: + # We add 1 element batch tt_a to a batch_size element batch tt_b to get + # the answer TensorTrainBatch of batch_size == tt_b.batch_size. + batch_size = shapes.lazy_batch_size(tt_b) + else: + batch_size = shapes.lazy_batch_size(tt_a) + tt_a = shapes.expand_batch_dim(tt_a) + tt_b = shapes.expand_batch_dim(tt_b) + tt_cores = [] + for core_idx in range(ndims): + a_core = tt_a.tt_cores[core_idx] + if tt_a.batch_size == 1: + a_core = tf.tile(a_core, (batch_size, 1, 1, 1, 1)) + b_core = tt_b.tt_cores[core_idx] + if tt_b.batch_size == 1: + b_core = tf.tile(b_core, (batch_size, 1, 1, 1, 1)) + if core_idx == 0: + curr_core = tf.concat((a_core, b_core), axis=4) + elif core_idx == ndims - 1: + curr_core = tf.concat((a_core, b_core), axis=1) + else: + upper_zeros = tf.zeros((batch_size, a_ranks[core_idx], shape[0][core_idx], + shape[1][core_idx], b_ranks[core_idx + 1]), dtype) + lower_zeros = tf.zeros((batch_size, b_ranks[core_idx], shape[0][core_idx], + shape[1][core_idx], a_ranks[core_idx + 1]), dtype) + upper = tf.concat((a_core, upper_zeros), axis=4) + lower = tf.concat((lower_zeros, b_core), axis=4) + curr_core = tf.concat((upper, lower), axis=1) + tt_cores.append(curr_core) + return tt_cores, batch_size + + +def add(tt_a, tt_b): + """Returns a TensorTrain corresponding to elementwise sum tt_a + tt_b. + + The shapes of tt_a and tt_b should coincide. + Supports broadcasting: + add(TensorTrainBatch, TensorTrain) + adds TensorTrain to each element in the batch of TTs in TensorTrainBatch. + + Args: + tt_a: `TensorTrain`, `TensorTrainBatch`, TT-tensor, or TT-matrix + tt_b: `TensorTrain`, `TensorTrainBatch`, TT-tensor, or TT-matrix + + Returns + a `TensorTrain` object corresponding to the element-wise sum of arguments if + both arguments are `TensorTrain`s. + OR a `TensorTrainBatch` if at least one of the arguments is + `TensorTrainBatch` + + Raises + ValueError if the arguments shapes do not coincide + """ + ndims = tt_a.ndims() + if tt_a.is_tt_matrix() != tt_b.is_tt_matrix(): + raise ValueError('The arguments should be both TT-tensors or both ' + 'TT-matrices') + + if tt_a.get_raw_shape() != tt_b.get_raw_shape(): + raise ValueError('The arguments should have the same shape.') + + if not shapes.is_batch_broadcasting_possible(tt_a, tt_b): + raise ValueError('The batch sizes are different and not 1, broadcasting is ' + 'not available.') + + is_batch_case = isinstance(tt_a, TensorTrainBatch) or isinstance(tt_b, TensorTrainBatch) + batch_size = None + if is_batch_case: + if tt_a.is_tt_matrix(): + tt_cores, batch_size = _add_batch_matrix_cores(tt_a, tt_b) + else: + tt_cores, batch_size = _add_batch_tensor_cores(tt_a, tt_b) + else: + if tt_a.is_tt_matrix(): + tt_cores = _add_matrix_cores(tt_a, tt_b) + else: + tt_cores = _add_tensor_cores(tt_a, tt_b) + + out_ranks = [1] + static_a_ranks = tt_a.get_tt_ranks() + static_b_ranks = tt_b.get_tt_ranks() + for core_idx in range(1, ndims): + out_ranks.append(static_a_ranks[core_idx] + static_b_ranks[core_idx]) + out_ranks.append(1) + if is_batch_case: + return TensorTrainBatch(tt_cores, tt_a.get_raw_shape(), out_ranks, + batch_size) + else: + return TensorTrain(tt_cores, tt_a.get_raw_shape(), out_ranks) + + +def multiply(tt_left, right): + """Returns a TensorTrain corresponding to element-wise product tt_left * right. + + Supports broadcasting: + multiply(TensorTrainBatch, TensorTrain) returns TensorTrainBatch consisting + of element-wise products of TT in TensorTrainBatch and TensorTrain + + multiply(TensorTrainBatch_a, TensorTrainBatch_b) returns TensorTrainBatch + consisting of element-wise products of TT in TensorTrainBatch_a and + TT in TensorTrainBatch_b + + Batch sizes should support broadcasting + Args: + tt_left: `TensorTrain` OR `TensorTrainBatch` + right: `TensorTrain` OR `TensorTrainBatch` OR a number. + + Returns + a `TensorTrain` or `TensorTrainBatch` object corresponding to the + element-wise product of the arguments. + + Raises + ValueError if the arguments shapes do not coincide or broadcasting is not + possible. + """ + + is_left_batch = isinstance(tt_left, TensorTrainBatch) + is_right_batch = isinstance(right, TensorTrainBatch) + + is_batch_case = is_left_batch or is_right_batch + ndims = tt_left.ndims() + if not isinstance(right, TensorTrainBase): + # Assume right is a number, not TensorTrain. + # To squash right uniformly across TT-cores we pull its absolute value + # and raise to the power 1/ndims. First TT-core is multiplied by the sign + # of right. + tt_cores = list(tt_left.tt_cores) + fact = tf.pow(tf.cast(tf.abs(right), tt_left.dtype), 1.0 / ndims) + sign = tf.cast(tf.sign(right), tt_left.dtype) + for i in range(len(tt_cores)): + tt_cores[i] = fact * tt_cores[i] + + tt_cores[0] = tt_cores[0] * sign + out_ranks = tt_left.get_tt_ranks() + if is_left_batch: + out_batch_size = tt_left.batch_size + else: + + if tt_left.is_tt_matrix() != right.is_tt_matrix(): + raise ValueError('The arguments should be both TT-tensors or both ' + 'TT-matrices') + + if tt_left.get_raw_shape() != right.get_raw_shape(): + raise ValueError('The arguments should have the same shape.') + + out_batch_size = 1 + dependencies = [] + can_determine_if_broadcast = True + if is_left_batch and is_right_batch: + if tt_left.batch_size is None and right.batch_size is None: + can_determine_if_broadcast = False + elif tt_left.batch_size is None and right.batch_size is not None: + if right.batch_size > 1: + can_determine_if_broadcast = False + elif tt_left.batch_size is not None and right.batch_size is None: + if tt_left.batch_size > 1: + can_determine_if_broadcast = False + + if not can_determine_if_broadcast: + # Cannot determine if broadcasting is needed. Avoid broadcasting and + # assume elementwise multiplication AND add execution time assert to print + # a better error message if the batch sizes turn out to be different. + + message = ('The batch sizes were unknown on compilation stage, so ' + 'assumed elementwise multiplication (i.e. no broadcasting). ' + 'Now it seems that they are different after all :') + + data = [message, shapes.lazy_batch_size(tt_left), ' x ', + shapes.lazy_batch_size(right)] + bs_eq = tf.assert_equal(shapes.lazy_batch_size(tt_left), + shapes.lazy_batch_size(right), data=data) + + dependencies.append(bs_eq) + + do_broadcast = shapes.is_batch_broadcasting_possible(tt_left, right) + if not can_determine_if_broadcast: + # Assume elementwise multiplication if broadcasting cannot be determined + # on compilation stage. + do_broadcast = False + if not do_broadcast and can_determine_if_broadcast: + raise ValueError('The batch sizes are different and not 1, broadcasting ' + 'is not available.') + + a_ranks = shapes.lazy_tt_ranks(tt_left) + b_ranks = shapes.lazy_tt_ranks(right) + shape = shapes.lazy_raw_shape(tt_left) + + output_str = '' + bs_str_left = '' + bs_str_right = '' + + if is_batch_case: + if is_left_batch and is_right_batch: + # Both arguments are batches of equal size. + if tt_left.batch_size == right.batch_size or not can_determine_if_broadcast: + bs_str_left = 'n' + bs_str_right = 'n' + output_str = 'n' + if not can_determine_if_broadcast: + out_batch_size = None + else: + out_batch_size = tt_left.batch_size + else: + # Broadcasting (e.g batch_sizes are 1 and n>1). + bs_str_left = 'n' + bs_str_right = 'm' + output_str = 'nm' + if tt_left.batch_size is None or tt_left.batch_size > 1: + out_batch_size = tt_left.batch_size + else: + out_batch_size = right.batch_size + else: + # One of the arguments is TensorTrain. + if is_left_batch: + bs_str_left = 'n' + bs_str_right = '' + out_batch_size = tt_left.batch_size + else: + bs_str_left = '' + bs_str_right = 'n' + out_batch_size = right.batch_size + output_str = 'n' + + is_matrix = tt_left.is_tt_matrix() + tt_cores = [] + + for core_idx in range(ndims): + a_core = tt_left.tt_cores[core_idx] + b_core = right.tt_cores[core_idx] + left_rank = a_ranks[core_idx] * b_ranks[core_idx] + right_rank = a_ranks[core_idx + 1] * b_ranks[core_idx + 1] + if is_matrix: + with tf.control_dependencies(dependencies): + curr_core = tf.einsum('{0}aijb,{1}cijd->{2}acijbd'.format(bs_str_left, + bs_str_right, output_str), a_core, b_core) + curr_core = tf.reshape(curr_core, (-1, left_rank, + shape[0][core_idx], + shape[1][core_idx], + right_rank)) + if not is_batch_case: + curr_core = tf.squeeze(curr_core, axis=0) + else: + with tf.control_dependencies(dependencies): + curr_core = tf.einsum('{0}aib,{1}cid->{2}acibd'.format(bs_str_left, + bs_str_right, output_str), a_core, b_core) + curr_core = tf.reshape(curr_core, (-1, left_rank, + shape[0][core_idx], right_rank)) + if not is_batch_case: + curr_core = tf.squeeze(curr_core, axis=0) + + tt_cores.append(curr_core) + + combined_ranks = zip(tt_left.get_tt_ranks(), right.get_tt_ranks()) + out_ranks = [a * b for a, b in combined_ranks] + + if not is_batch_case: + return TensorTrain(tt_cores, tt_left.get_raw_shape(), out_ranks) + else: + return TensorTrainBatch(tt_cores, tt_left.get_raw_shape(), out_ranks, + batch_size=out_batch_size) + +def frobenius_norm_squared(tt, differentiable=False): + """Frobenius norm squared of `TensorTrain` or of each TT in `TensorTrainBatch`. + + Frobenius norm squared is the sum of squares of all elements in a tensor. + + Args: + tt: `TensorTrain` or `TensorTrainBatch` object + differentiable: bool, whether to use a differentiable implementation + or a fast and stable implementation based on QR decomposition. + + Returns + a number which is the Frobenius norm squared of `tt`, if it is `TensorTrain` + OR + a Tensor of size tt.batch_size, consisting of the Frobenius norms squared of + each TensorTrain in `tt`, if it is `TensorTrainBatch` + """ + if differentiable: + if hasattr(tt, 'batch_size'): + bs_str = 'n' + else: + bs_str = '' + if tt.is_tt_matrix(): + running_prod = tf.einsum('{0}aijb,{0}cijd->{0}bd'.format(bs_str), + tt.tt_cores[0], tt.tt_cores[0]) + else: + running_prod = tf.einsum('{0}aib,{0}cid->{0}bd'.format(bs_str), + tt.tt_cores[0], tt.tt_cores[0]) + + for core_idx in range(1, tt.ndims()): + curr_core = tt.tt_cores[core_idx] + if tt.is_tt_matrix(): + running_prod = tf.einsum('{0}ac,{0}aijb,{0}cijd->{0}bd'.format(bs_str), + running_prod, curr_core, curr_core) + else: + running_prod = tf.einsum('{0}ac,{0}aib,{0}cid->{0}bd'.format(bs_str), + running_prod, curr_core, curr_core) + + return tf.squeeze(running_prod, [-1, -2]) + + else: + orth_tt = decompositions.orthogonalize_tt_cores(tt, left_to_right=True) + # All the cores of orth_tt except the last one are orthogonal, hence + # the Frobenius norm of orth_tt equals to the norm of the last core. + if hasattr(tt, 'batch_size'): + batch_size = shapes.lazy_batch_size(tt) + last_core = tf.reshape(orth_tt.tt_cores[-1], (batch_size, -1)) + return tf.norm(last_core, axis=1) ** 2 + else: + return tf.norm(orth_tt.tt_cores[-1]) ** 2 + + +def frobenius_norm(tt, epsilon=1e-5, differentiable=False): + """Frobenius norm of `TensorTrain` or of each TT in `TensorTrainBatch` + + Frobenius norm is the sqrt of the sum of squares of all elements in a tensor. + + Args: + tt: `TensorTrain` or `TensorTrainBatch` object + epsilon: the function actually computes sqrt(norm_squared + epsilon) for + numerical stability (e.g. gradient of sqrt at zero is inf). + differentiable: bool, whether to use a differentiable implementation or + a fast and stable implementation based on QR decomposition. + + Returns + a number which is the Frobenius norm of `tt`, if it is `TensorTrain` + OR + a Tensor of size tt.batch_size, consisting of the Frobenius norms of + each TensorTrain in `tt`, if it is `TensorTrainBatch` + """ + return tf.sqrt(frobenius_norm_squared(tt, differentiable) + epsilon) + + +def transpose(tt_matrix): + """Transpose a TT-matrix or a batch of TT-matrices. + + Args: + tt_matrix: `TensorTrain` or `TensorTrainBatch` object containing a TT-matrix + (or a batch of TT-matrices). + + Returns: + `TensorTrain` or `TensorTrainBatch` object containing a transposed TT-matrix + (or a batch of TT-matrices). + + Raises: + ValueError if the argument is not a TT-matrix. + """ + if not isinstance(tt_matrix, TensorTrainBase) or not tt_matrix.is_tt_matrix(): + raise ValueError('The argument should be a TT-matrix.') + + transposed_tt_cores = [] + for core_idx in range(tt_matrix.ndims()): + curr_core = tt_matrix.tt_cores[core_idx] + if isinstance(tt_matrix, TensorTrain): + transposed_tt_cores.append(tf.transpose(curr_core, (0, 2, 1, 3))) + else: + # TensorTrainBatch. + transposed_tt_cores.append(tf.transpose(curr_core, (0, 1, 3, 2, 4))) + + tt_matrix_shape = tt_matrix.get_raw_shape() + transposed_shape = tt_matrix_shape[1], tt_matrix_shape[0] + tt_ranks = tt_matrix.get_tt_ranks() + if isinstance(tt_matrix, TensorTrain): + return TensorTrain(transposed_tt_cores, transposed_shape, tt_ranks) + else: + batch_size = tt_matrix.batch_size + return TensorTrainBatch(transposed_tt_cores, transposed_shape, tt_ranks, + batch_size) + + +def quadratic_form(A, b, c): + """Quadratic form b^t A c; A is a TT-matrix, b and c can be batches. + + Args: + A: `TensorTrain` object containing a TT-matrix of size N x M. + b: `TensorTrain` object containing a TT-matrix of size N x 1 + or `TensorTrainBatch` with a batch of TT-matrices of size N x 1. + c: `TensorTrain` object containing a TT-matrix of size M x 1 + or `TensorTrainBatch` with a batch of TT-matrices of size M x 1. + + Returns: + A number, the value of the quadratic form if all the arguments are + `TensorTrain`s. + OR tf.Tensor of size batch_size if at least one of the arguments is + `TensorTrainBatch` + + Raises: + ValueError if the arguments are not TT-matrices or if the shapes are + not consistent. + """ + if not isinstance(A, TensorTrainBase) or not A.is_tt_matrix(): + raise ValueError('The arguments should be a TT-matrix.') + + # TODO: support tf.Tensor as b and c. + if not isinstance(b, TensorTrainBase) or not b.is_tt_matrix(): + raise ValueError('The arguments should be a TT-matrix.') + if not isinstance(c, TensorTrainBase) or not c.is_tt_matrix(): + raise ValueError('The arguments should be a TT-matrix.') + + b_is_batch = isinstance(b, TensorTrainBatch) + c_is_batch = isinstance(b, TensorTrainBatch) + b_bs_str = 'p' if b_is_batch else '' + c_bs_str = 'p' if c_is_batch else '' + out_bs_str = 'p' if b_is_batch or c_is_batch else '' + + ndims = A.ndims() + curr_core_1 = b.tt_cores[0] + curr_core_2 = c.tt_cores[0] + curr_matrix_core = A.tt_cores[0] + # We enumerate the dummy dimension (that takes 1 value) with `k`. + # You may think that using two different k would be faster, but in my + # experience it's even a little bit slower (but neglectable in general). + einsum_str = '{0}aikb,cijd,{1}ejkf->{2}bdf'.format(b_bs_str, c_bs_str, + out_bs_str) + res = tf.einsum(einsum_str, curr_core_1, curr_matrix_core, curr_core_2) + for core_idx in range(1, ndims): + curr_core_1 = b.tt_cores[core_idx] + curr_core_2 = c.tt_cores[core_idx] + curr_matrix_core = A.tt_cores[core_idx] + einsum_str = '{2}ace,{0}aikb,cijd,{1}ejkf->{2}bdf'.format(b_bs_str, + c_bs_str, + out_bs_str) + res = tf.einsum(einsum_str, res, curr_core_1, + curr_matrix_core, curr_core_2) + + # Squeeze to make the result a number instead of 1 x 1 for NON batch case and + # to make the result a tensor of size + # batch_size + # instead of + # batch_size x 1 x 1 + # in the batch case. + return tf.squeeze(res) + + +def cast(tt_a, dtype): + """Casts a tt-tensor to a new type. + + Args: + tt_a: `TensorTrain` object. + dtype: The destination type. + + Raises: + TypeError: If `tt_a` cannot be cast to the `dtype`. + ValueError: If `tt_a` is not a `TensorTrain` or `TensorTrainBatch`. + """ + res_cores = [] + cores = tt_a.tt_cores + for core_idx in range(tt_a.ndims()): + res_cores.append(tf.cast(cores[core_idx], dtype)) + res_shape = tt_a.get_raw_shape() + res_ranks = tt_a.get_tt_ranks() + if isinstance(tt_a, TensorTrain): + return TensorTrain(res_cores, res_shape, res_ranks) + elif isinstance(tt_a, TensorTrainBatch): + return TensorTrainBatch(res_cores, res_shape, res_ranks, tt_a.batch_size) + else: + raise ValueError('Unsupported type of input "%s", should be TensorTrain or ' + 'TensorTrainBatch.' % tt_a) + + +def gather_nd(tt, indices): + """out[i] = tt[indices[i, 0], indices[i, 1], ...] + + Equivalent to + tf.gather_nd(t3f.full(tt), indices) + but much faster, since it does not materialize the full tensor. + + For batches of TT works indices should include the batch dimension as well. + + Args: + tt: `TensorTrain` or `TensorTrainBatch` object representing a tensor + (TT-matrices are not implemented yet) + indices: numpy array, tf.Tensor, placeholder with 2 or more dimensions. + The last dimension indices.shape[-1] should be equal to the numbers of + dimensions in TT: + indices.shape[-1] = tt.ndims for `TensorTrain` + indices.shape[-1] = tt.ndims + 1 for `TensorTrainBatch` + + Returns: + tf.Tensor with elements specified by indices. + + Raises: + ValueError if `indices` have wrong shape. + NotImplementedError if `tt` is a TT-matrix. + """ + if tt.is_tt_matrix(): + raise NotImplementedError('gather_nd doesnt support TT-matrices yet ' + '(got %s)' % tt) + indices = tf.convert_to_tensor(indices) + if isinstance(tt, TensorTrainBatch): + if indices.get_shape()[-1] != tt.ndims() + 1: + raise ValueError('The last dimension of indices (%d) should have ' + 'the same size as the number of dimensions in the tt ' + 'object (%d) + 1 (for the batch dimension).' % + (indices.get_shape()[-1], tt.ndims())) + else: + if indices.get_shape()[-1] != tt.ndims(): + raise ValueError('The last dimension of indices (%d) should have ' + 'the same size as the number of dimensions in the tt ' + 'object (%d).' % (indices.get_shape()[-1], tt.ndims())) + tt_elements = tf.ones(tf.shape(indices)[:-1]) + tt_elements = tf.reshape(tt_elements, (-1, 1, 1)) + for core_idx in range(tt.ndims()): + curr_core = tt.tt_cores[core_idx] + if isinstance(tt, TensorTrainBatch): + curr_core = tf.transpose(curr_core, (0, 2, 1, 3)) + curr_idx = tf.stack((indices[:, 0], indices[:, core_idx + 1]), axis=1) + core_slices = tf.gather_nd(curr_core, curr_idx) + else: + curr_core = tf.transpose(curr_core, (1, 0, 2)) + core_slices = tf.gather(curr_core, indices[:, core_idx]) + tt_elements = tf.matmul(tt_elements, core_slices) + tt_elements = tf.reshape(tt_elements, tf.shape(indices)[:-1]) + return tt_elements + + +def renormalize_tt_cores(tt, epsilon=1e-8): + """Renormalizes TT-cores to make them of the same Frobenius norm. + + Doesn't change the tensor represented by `tt` object, but renormalizes the + TT-cores to make further computations more stable. + + Args: + tt: `TensorTrain` or `TensorTrainBatch` object + epsilon: parameter for numerical stability of sqrt + Returns: + `TensorTrain` or `TensorTrainBatch` which represents the same + tensor as tt, but with all cores having equal norm. In the batch + case applies to each TT in `TensorTrainBatch`. + + """ + if isinstance(tt, TensorTrain): + new_cores = [] + running_log_norm = 0 + core_norms = [] + for core in tt.tt_cores: + cur_core_norm = tf.sqrt(tf.maximum(tf.reduce_sum(core ** 2), epsilon)) + core_norms.append(cur_core_norm) + running_log_norm += tf.log(cur_core_norm) + + running_log_norm = running_log_norm / tt.ndims() + fact = tf.exp(running_log_norm) + for i, core in enumerate(tt.tt_cores): + new_cores.append(core * fact / core_norms[i]) + + return TensorTrain(new_cores) + else: + sz = (tt.batch_size,) + (len(tt.tt_cores[0].shape) - 1) * (1,) + running_core_log_norms = tf.zeros(sz) + ax = np.arange(len(tt.tt_cores[0].shape))[1:] + fact_list = [] + for core in tt.tt_cores: + cur_core_norm_sq = tf.reduce_sum(core**2, axis=ax, keep_dims=True) + cur_core_norm = tf.sqrt(tf.maximum(epsilon, cur_core_norm_sq)) + fact_list.append(cur_core_norm) + running_core_log_norms += tf.log(cur_core_norm) + + new_cores = [] + exp_fact = tf.exp(running_core_log_norms / tt.ndims()) + for i, core in enumerate(tt.tt_cores): + new_cores.append(tf.multiply(core, exp_fact / fact_list[i])) + + return TensorTrainBatch(new_cores) diff --git a/build/lib/t3f/ops_test.py b/build/lib/t3f/ops_test.py new file mode 100644 index 00000000..cb1dcc7c --- /dev/null +++ b/build/lib/t3f/ops_test.py @@ -0,0 +1,880 @@ +import numpy as np +import tensorflow as tf + +from t3f.tensor_train import TensorTrain +from t3f.tensor_train_batch import TensorTrainBatch +from t3f import ops +from t3f import shapes +from t3f import initializers + + +class TTTensorTest(tf.test.TestCase): + + def testFullTensor2d(self): + np.random.seed(1) + for rank in [1, 2]: + a = np.random.rand(10, rank).astype(np.float32) + b = np.random.rand(rank, 9).astype(np.float32) + tt_cores = (a.reshape(1, 10, rank), b.reshape(rank, 9, 1)) + desired = np.dot(a, b) + with self.test_session(): + tf_tens = TensorTrain(tt_cores) + actual = ops.full(tf_tens) + self.assertAllClose(desired, actual.eval()) + + def testFullTensor3d(self): + np.random.seed(1) + for rank_1 in [1, 2]: + a = np.random.rand(10, rank_1).astype(np.float32) + b = np.random.rand(rank_1, 9, 3).astype(np.float32) + c = np.random.rand(3, 8).astype(np.float32) + tt_cores = (a.reshape(1, 10, rank_1), b, c.reshape((3, 8, 1))) + # Basically do full by hand. + desired = a.dot(b.reshape((rank_1, -1))) + desired = desired.reshape((-1, 3)).dot(c) + desired = desired.reshape(10, 9, 8) + with self.test_session(): + tf_tens = TensorTrain(tt_cores) + actual = ops.full(tf_tens) + self.assertAllClose(desired, actual.eval()) + + def testFlatInnerTTTensbyTTTens(self): + # Inner product between two TT-tensors. + shape_list = ((2, 2), + (2, 3, 4), + (4, 2, 5, 2)) + rank_list = (1, 2) + with self.test_session() as sess: + for shape in shape_list: + for rank in rank_list: + tt_1 = initializers.random_tensor(shape, tt_rank=rank) + tt_2 = initializers.random_tensor(shape, tt_rank=rank) + res_actual = ops.flat_inner(tt_1, tt_2) + tt_1_full = tf.reshape(ops.full(tt_1), (1, -1)) + tt_2_full = tf.reshape(ops.full(tt_2), (-1, 1)) + res_desired = tf.matmul(tt_1_full, tt_2_full) + res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) + self.assertAllClose(res_actual_val, np.squeeze(res_desired_val), + rtol=1e-5) + + def testFlatInnerTTTensbySparseTens(self): + # Inner product between a TT-tensor and a sparse tensor. + shape_list = ((2, 2), + (2, 3, 4), + (4, 2, 5, 2)) + rank_list = (1, 2) + np.random.seed(1) + with self.test_session() as sess: + for shape in shape_list: + for rank in rank_list: + for num_elements in [1, 10]: + tt_1 = initializers.random_tensor(shape, tt_rank=rank) + sparse_flat_indices = np.random.choice(np.prod(shape), num_elements).astype(int) + sparse_indices = np.unravel_index(sparse_flat_indices, shape) + sparse_indices = np.vstack(sparse_indices).transpose() + values = np.random.randn(num_elements).astype(np.float32) + sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, + dense_shape=shape) + res_actual = ops.flat_inner(tt_1, sparse_2) + res_actual_val, tt_1_val = sess.run([res_actual, ops.full(tt_1)]) + res_desired_val = tt_1_val.flatten()[sparse_flat_indices].dot(values) + self.assertAllClose(res_actual_val, res_desired_val) + + def testAdd(self): + # Sum two TT-tensors. + tt_a = initializers.random_tensor((2, 1, 3, 4), tt_rank=2) + tt_b = initializers.random_tensor((2, 1, 3, 4), tt_rank=[1, 2, 4, 3, 1]) + with self.test_session() as sess: + res_actual = ops.full(ops.add(tt_a, tt_b)) + res_actual2 = ops.full(tt_a + tt_b) + res_desired = ops.full(tt_a) + ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testMultiply(self): + # Multiply two TT-tensors. + tt_a = initializers.random_tensor((1, 2, 3, 4), tt_rank=2) + tt_b = initializers.random_tensor((1, 2, 3, 4), tt_rank=[1, 1, 4, 3, 1]) + with self.test_session() as sess: + res_actual = ops.full(ops.multiply(tt_a, tt_b)) + res_actual2 = ops.full(tt_a * tt_b) + res_desired = ops.full(tt_a) * ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testMultiplyByNumber(self): + # Multiply a tensor by a number. + tt = initializers.random_tensor((1, 2, 3), tt_rank=(1, 2, 3, 1)) + with self.test_session() as sess: + res_actual = ops.full(ops.multiply(tt, 4)) + res_actual2 = ops.full(4.0 * tt) + res_desired = 4.0 * ops.full(tt) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testFrobeniusNormTens(self): + # Frobenius norm of a TT-tensor. + shape_list = ((2, 2), + (2, 3, 4), + (4, 2, 5, 2)) + rank_list = (1, 2) + with self.test_session() as sess: + for shape in shape_list: + for rank in rank_list: + tt = initializers.random_tensor(shape, tt_rank=rank) + norm_sq_actual = ops.frobenius_norm_squared(tt) + norm_actual = ops.frobenius_norm(tt) + vars = [norm_sq_actual, norm_actual, ops.full(tt)] + norm_sq_actual_val, norm_actual_val, tt_val = sess.run(vars) + tt_val = tt_val.flatten() + norm_sq_desired_val = tt_val.dot(tt_val) + norm_desired_val = np.linalg.norm(tt_val) + self.assertAllClose(norm_sq_actual_val, norm_sq_desired_val) + self.assertAllClose(norm_actual_val, norm_desired_val, atol=1e-5, + rtol=1e-5) + + def testCastFloat(self): + # Test cast function for float tt-tensors. + tt_x = initializers.random_tensor((2, 3, 2), tt_rank=2) + + with self.test_session() as sess: + for dtype in [tf.float16, tf.float32, tf.float64]: + casted = ops.cast(tt_x, dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(dtype, casted.dtype) + self.assertTrue(dtype, casted_val.dtype) + + def testCastIntFloat(self): + # Tests cast function from int to float for tensors. + np.random.seed(1) + K_1 = np.random.randint(0, high=100, size=(1, 2, 2)) + K_2 = np.random.randint(0, high=100, size=(2, 3, 2)) + K_3 = np.random.randint(0, high=100, size=(2, 2, 1)) + tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) + + with self.test_session() as sess: + for dtype in [tf.float16, tf.float32, tf.float64]: + casted = ops.cast(tt_int, dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(dtype, casted.dtype) + self.assertTrue(dtype, casted_val.dtype) + + def testCoreRenorm(self): + a = initializers.random_tensor(3 * (10,), tt_rank=7) + b = ops.renormalize_tt_cores(a) + var_list = [ops.full(a), ops.full(b)] + with self.test_session() as sess: + af, bf = sess.run(var_list) + b_cores = sess.run(b.tt_cores) + b_cores_norms = [] + for cr in b_cores: + b_cores_norms.append(np.linalg.norm(cr)) + self.assertAllClose(af, bf, atol=1e-5, rtol=1e-5) + self.assertAllClose(b_cores_norms, b_cores_norms[0] + * np.ones((len(b_cores)))) + + +class TTMatrixTest(tf.test.TestCase): + + def testFullMatrix2d(self): + np.random.seed(1) + for rank in [1, 2]: + a = np.random.rand(2, 3, rank).astype(np.float32) + b = np.random.rand(rank, 4, 5).astype(np.float32) + tt_cores = (a.reshape(1, 2, 3, rank), b.reshape((rank, 4, 5, 1))) + # Basically do full by hand. + desired = a.reshape((-1, rank)).dot(b.reshape((rank, -1))) + desired = desired.reshape((2, 3, 4, 5)) + desired = desired.transpose((0, 2, 1, 3)) + desired = desired.reshape((2 * 4, 3 * 5)) + with self.test_session(): + tf_mat = TensorTrain(tt_cores) + actual = ops.full(tf_mat) + self.assertAllClose(desired, actual.eval()) + + def testFullMatrix3d(self): + np.random.seed(1) + for rank in [1, 2]: + a = np.random.rand(2, 3, rank).astype(np.float32) + b = np.random.rand(rank, 4, 5, rank).astype(np.float32) + c = np.random.rand(rank, 2, 2).astype(np.float32) + tt_cores = (a.reshape(1, 2, 3, rank), b.reshape(rank, 4, 5, rank), + c.reshape(rank, 2, 2, 1)) + # Basically do full by hand. + desired = a.reshape((-1, rank)).dot(b.reshape((rank, -1))) + desired = desired.reshape((-1, rank)).dot(c.reshape((rank, -1))) + desired = desired.reshape((2, 3, 4, 5, 2, 2)) + desired = desired.transpose((0, 2, 4, 1, 3, 5)) + desired = desired.reshape((2 * 4 * 2, 3 * 5 * 2)) + with self.test_session(): + tf_mat = TensorTrain(tt_cores) + actual = ops.full(tf_mat) + self.assertAllClose(desired, actual.eval()) + + def testTTMatTimesTTMat(self): + # Multiply a TT-matrix by another TT-matrix. + left_shape = (2, 3, 4) + sum_shape = (4, 3, 5) + right_shape = (4, 4, 4) + with self.test_session() as sess: + tt_mat_1 = initializers.random_matrix((left_shape, sum_shape), tt_rank=3) + tt_mat_2 = initializers.random_matrix((sum_shape, right_shape)) + res_actual = ops.matmul(tt_mat_1, tt_mat_2) + res_actual = ops.full(res_actual) + res_desired = tf.matmul(ops.full(tt_mat_1), ops.full(tt_mat_2)) + res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) + # TODO: why so bad accuracy? + self.assertAllClose(res_actual_val, res_desired_val, atol=1e-4, rtol=1e-4) + + def testTTMatTimesDenseVec(self): + # Multiply a TT-matrix by a dense vector. + inp_shape = (2, 3, 4) + out_shape = (3, 4, 3) + np.random.seed(1) + vec = np.random.rand(np.prod(inp_shape), 1).astype(np.float32) + with self.test_session() as sess: + tf_vec = tf.constant(vec) + tf.set_random_seed(1) + tt_mat = initializers.random_matrix((out_shape, inp_shape)) + res_actual = ops.matmul(tt_mat, tf_vec) + res_desired = tf.matmul(ops.full(tt_mat), tf_vec) + res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) + self.assertAllClose(res_actual_val, res_desired_val) + + def testDenseMatTimesTTVec(self): + # Multiply a TT-matrix by a dense vector. + inp_shape = (3, 3, 3, 3) + out_shape = (3, 3, 3, 3) + np.random.seed(1) + mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)).astype(np.float32) + with self.test_session() as sess: + tf_mat = tf.constant(mat) + tf.set_random_seed(1) + tt_vec = initializers.random_matrix((inp_shape, None)) + res_actual = ops.matmul(tf_mat, tt_vec) + res_desired = tf.matmul(tf_mat, ops.full(tt_vec)) + res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) + self.assertAllClose(res_actual_val, res_desired_val, atol=1e-4, rtol=1e-4) + + def testFlatInnerTTMatbyTTMat(self): + # Inner product between two TT-Matrices. + shape_list = (((2, 2), (3, 4)), + ((2, 3, 4), (2, 2, 2))) + rank_list = (1, 2) + with self.test_session() as sess: + for shape in shape_list: + for rank in rank_list: + tt_1 = initializers.random_matrix(shape, tt_rank=rank) + tt_2 = initializers.random_matrix(shape, tt_rank=rank) + res_actual = ops.flat_inner(tt_1, tt_2) + tt_1_full = tf.reshape(ops.full(tt_1), (1, -1)) + tt_2_full = tf.reshape(ops.full(tt_2), (-1, 1)) + res_desired = tf.matmul(tt_1_full, tt_2_full) + res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) + self.assertAllClose(res_actual_val, np.squeeze(res_desired_val), + rtol=1e-5, atol=1e-5) + + def testFlatInnerTTMatbySparseMat(self): + # Inner product between a TT-matrix and a sparse matrix. + shape_list = (((2, 2), (3, 4)), + ((2, 3, 4), (2, 2, 2))) + rank_list = (1, 2) + np.random.seed(1) + with self.test_session() as sess: + for tensor_shape in shape_list: + for rank in rank_list: + for num_elements in [1, 9]: + tt_1 = initializers.random_matrix(tensor_shape, tt_rank=rank) + matrix_shape = np.prod(tensor_shape[0]), np.prod(tensor_shape[1]) + sparse_flat_indices = np.random.choice(np.prod(matrix_shape), num_elements) + sparse_flat_indices = sparse_flat_indices.astype(int) + sparse_indices = np.unravel_index(sparse_flat_indices, matrix_shape) + sparse_indices = np.vstack(sparse_indices).transpose() + values = np.random.randn(num_elements).astype(np.float32) + sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, + dense_shape=matrix_shape) + res_actual = ops.flat_inner(tt_1, sparse_2) + res_actual_val, tt_1_val = sess.run([res_actual, ops.full(tt_1)]) + res_desired_val = tt_1_val.flatten()[sparse_flat_indices].dot(values) + self.assertAllClose(res_actual_val, res_desired_val) + + def testFrobeniusNormMatrix(self): + # Frobenius norm of a TT-matrix. + shape_list = (((2, 2), (3, 4)), + ((2, 3, 4), (2, 2, 2))) + rank_list = (1, 2) + with self.test_session() as sess: + for tensor_shape in shape_list: + for rank in rank_list: + tt = initializers.random_matrix(tensor_shape, tt_rank=rank) + norm_sq_actual = ops.frobenius_norm_squared(tt) + norm_actual = ops.frobenius_norm(tt) + vars = [norm_sq_actual, norm_actual, ops.full(tt)] + norm_sq_actual_val, norm_actual_val, tt_val = sess.run(vars) + tt_val = tt_val.flatten() + norm_sq_desired_val = tt_val.dot(tt_val) + norm_desired_val = np.linalg.norm(tt_val) + self.assertAllClose(norm_sq_actual_val, norm_sq_desired_val) + self.assertAllClose(norm_actual_val, norm_desired_val, atol=1e-5, + rtol=1e-5) + + def testTranspose(self): + # Transpose a TT-matrix. + shape_list = (((2, 2), (3, 4)), + ((2, 3, 4), (2, 2, 2))) + rank_list = (1, 2) + with self.test_session() as sess: + for tensor_shape in shape_list: + for rank in rank_list: + tt = initializers.random_matrix(tensor_shape, tt_rank=rank) + res_actual = ops.full(ops.transpose(tt)) + res_actual_val, tt_val = sess.run([res_actual, ops.full(tt)]) + self.assertAllClose(tt_val.transpose(), res_actual_val) + + def testQuadraticForm(self): + # Test quadratic form. + shape_list = (((2, 2), (3, 4)), + ((2, 3, 4), (2, 2, 2))) + rank_list = (1, 2) + with self.test_session() as sess: + for tensor_shape in shape_list: + for rank in rank_list: + A = initializers.random_matrix(tensor_shape, tt_rank=rank) + b = initializers.random_matrix((tensor_shape[0], None), tt_rank=rank) + c = initializers.random_matrix((tensor_shape[1], None), tt_rank=rank) + res_actual = ops.quadratic_form(A, b, c) + vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] + res_actual_val, A_val, b_val, c_val = sess.run(vars) + res_desired = b_val.T.dot(A_val).dot(c_val) + self.assertAllClose(res_actual_val, np.squeeze(res_desired), + atol=1e-5, rtol=1e-5) + + def testQuadraticFormBatch(self): + # Test quadratic form for batch of tensors. + shape_list = (((2, 2), (3, 4)), + ((2, 3, 4), (2, 2, 2))) + rank_list = (1, 2) + with self.test_session() as sess: + for tensor_shape in shape_list: + for rank in rank_list: + A = initializers.random_matrix(tensor_shape, tt_rank=rank) + b = initializers.random_matrix_batch((tensor_shape[0], None), + tt_rank=rank, batch_size=5) + c = initializers.random_matrix_batch((tensor_shape[1], None), + tt_rank=rank, batch_size=5) + res_actual = ops.quadratic_form(A, b, c) + vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] + res_actual_val, A_val, b_val, c_val = sess.run(vars) + res_desired = np.diag(b_val[:, :, 0].dot(A_val).dot(c_val[:, :, 0].T)) + self.assertAllClose(res_actual_val, np.squeeze(res_desired), + atol=1e-5, rtol=1e-5) + + def testCastFloat(self): + # Test cast function for float tt-matrices and vectors. + + tt_mat = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=2) + tt_vec = initializers.random_matrix(((2, 3), None), tt_rank=2) + + with self.test_session() as sess: + for tt in [tt_mat, tt_vec]: + for dtype in [tf.float16, tf.float32, tf.float64]: + casted = ops.cast(tt, dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(dtype, casted.dtype) + self.assertTrue(dtype, casted_val.dtype) + + def testCastIntFloat(self): + # Tests cast function from int to float for matrices. + np.random.seed(1) + K_1 = np.random.randint(0, high=100, size=(1, 2, 2, 2)) + K_2 = np.random.randint(0, high=100, size=(2, 3, 3, 2)) + K_3 = np.random.randint(0, high=100, size=(2, 2, 2, 1)) + tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) + + with self.test_session() as sess: + for dtype in [tf.float16, tf.float32, tf.float64]: + casted = ops.cast(tt_int, dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(dtype, casted.dtype) + self.assertTrue(dtype, casted_val.dtype) + + def testUnknownRanksTTMatmul(self): + # Tests tt_tt_matmul for matrices with unknown ranks + K_1 = tf.placeholder(tf.float32, (1, 2, 2, None)) + K_2 = tf.placeholder(tf.float32, (None, 3, 3, 1)) + tt_mat = TensorTrain([K_1, K_2]) + res_actual = ops.full(ops.matmul(tt_mat, tt_mat)) + res_desired = tf.matmul(ops.full(tt_mat), ops.full(tt_mat)) + np.random.seed(1) + K_1_val = np.random.rand(1, 2, 2, 2) + K_2_val = np.random.rand(2, 3, 3, 1) + with self.test_session() as sess: + res_actual_val = sess.run(res_actual, {K_1: K_1_val, K_2: K_2_val}) + res_desired_val = sess.run(res_desired, {K_1: K_1_val, K_2: K_2_val}) + self.assertAllClose(res_desired_val, res_actual_val) + + + def testHalfKnownRanksTTMatmul(self): + # Tests tt_tt_matmul for the case when one matrice has known ranks + # and the other one doesn't + np.random.seed(1) + K_1 = tf.placeholder(tf.float32, (1, 2, 2, None)) + K_2 = tf.placeholder(tf.float32, (None, 3, 3, 1)) + tt_mat_known_ranks = TensorTrain([K_1, K_2], tt_ranks=[1, 3, 1]) + tt_mat = TensorTrain([K_1, K_2]) + res_actual = ops.full(ops.matmul(tt_mat_known_ranks, tt_mat)) + res_desired = tf.matmul(ops.full(tt_mat_known_ranks), ops.full(tt_mat)) + np.random.seed(1) + K_1_val = np.random.rand(1, 2, 2, 3) + K_2_val = np.random.rand(3, 3, 3, 1) + with self.test_session() as sess: + res_actual_val = sess.run(res_actual, {K_1: K_1_val, K_2: K_2_val}) + res_desired_val = sess.run(res_desired, {K_1: K_1_val, K_2: K_2_val}) + self.assertAllClose(res_desired_val, res_actual_val) + + +class TTTensorBatchTest(tf.test.TestCase): + + def testFullTensor2d(self): + np.random.seed(1) + for rank in [1, 2]: + a = np.random.rand(3, 10, rank).astype(np.float32) + b = np.random.rand(3, rank, 9).astype(np.float32) + tt_cores = (a.reshape(3, 1, 10, rank), b.reshape(3, rank, 9, 1)) + desired = np.einsum('oib,obj->oij', a, b) + with self.test_session(): + tf_tens = TensorTrainBatch(tt_cores) + actual = ops.full(tf_tens) + self.assertAllClose(desired, actual.eval()) + + def testFullTensor3d(self): + np.random.seed(1) + for rank_1 in [1, 2]: + a = np.random.rand(3, 10, rank_1).astype(np.float32) + b = np.random.rand(3, rank_1, 9, 3).astype(np.float32) + c = np.random.rand(3, 3, 8).astype(np.float32) + tt_cores = (a.reshape(3, 1, 10, rank_1), b, c.reshape((3, 3, 8, 1))) + # Basically do full by hand. + desired = np.einsum('oia,oajb,obk->oijk', a, b, c) + with self.test_session(): + tf_tens = TensorTrainBatch(tt_cores) + actual = ops.full(tf_tens) + self.assertAllClose(desired, actual.eval()) + + def testFlatInnerTTTensbyTTTensSameBatchSize(self): + # Inner product between two batch TT-tensors of the same batch_size. + shape_list = ((2, 2), + (2, 3, 4)) + rank_list = (1, 2) + with self.test_session() as sess: + for shape in shape_list: + for rank in rank_list: + tt_1 = initializers.random_tensor_batch(shape, tt_rank=rank, + batch_size=2) + tt_2 = initializers.random_tensor_batch(shape, tt_rank=rank, + batch_size=2) + res_actual = ops.flat_inner(tt_1, tt_2) + tt_1_full = tf.reshape(ops.full(tt_1), (2, 1, -1)) + tt_2_full = tf.reshape(ops.full(tt_2), (2, -1, 1)) + res_desired = tf.matmul(tt_1_full, tt_2_full) + res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) + self.assertAllClose(res_actual_val, np.squeeze(res_desired_val)) + + def testFlatInnerTTTensbyTTTensBroadcasting(self): + # Inner product between two batch TT-tensors with broadcasting. + tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=1) + tt_2 = initializers.random_tensor_batch((2, 3, 4), batch_size=3) + res_actual_1 = ops.flat_inner(tt_1, tt_2) + res_actual_2 = ops.flat_inner(tt_2, tt_1) + res_desired = tf.einsum('ijk,oijk->o', ops.full(tt_1[0]), ops.full(tt_2)) + with self.test_session() as sess: + res = sess.run([res_actual_1, res_actual_2, res_desired]) + res_actual_1_val, res_actual_2_val, res_desired_val = res + self.assertAllClose(res_actual_1_val, res_desired_val) + self.assertAllClose(res_actual_2_val, res_desired_val) + + tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=2) + with self.assertRaises(ValueError): + # The batch_sizes are different. + ops.flat_inner(tt_1, tt_2) + + def testAddSameBatchSize(self): + # Sum two TT-tensors with the same batch size. + tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=3) + tt_b = initializers.random_tensor_batch((2, 1, 4), tt_rank=[1, 2, 4, 1], + batch_size=3) + with self.test_session() as sess: + res_actual = ops.full(ops.add(tt_a, tt_b)) + res_actual2 = ops.full(tt_a + tt_b) + res_desired = ops.full(tt_a) + ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testAddBroadcasting(self): + # Sum two TT-tensors with broadcasting. + tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=1) + tt_b = initializers.random_tensor_batch((2, 1, 4), tt_rank=[1, 2, 4, 1], + batch_size=3) + with self.test_session() as sess: + res_actual = ops.full(ops.add(tt_a, tt_b)) + res_actual2 = ops.full(tt_b + tt_a) + res_desired = ops.full(tt_a) + ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testMultiplyByNumber(self): + # Multiply batch of tensors by a number. + tt = initializers.random_tensor_batch((1, 2, 3), tt_rank=(1, 2, 3, 1), + batch_size=3) + with self.test_session() as sess: + res_actual = ops.full(ops.multiply(tt, 4)) + res_actual2 = ops.full(4.0 * tt) + res_desired = 4.0 * ops.full(tt) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testFrobeniusNormDifferentiableBatch(self): + with self.test_session() as sess: + tt = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) + norm_sq_diff = ops.frobenius_norm_squared(tt, differentiable=True) + variables = [norm_sq_diff, ops.full(tt)] + norm_sq_diff_val, tt_full = sess.run(variables) + desired_norm = np.linalg.norm(tt_full.reshape((5, -1)), axis=1)**2 + self.assertAllClose(norm_sq_diff_val, desired_norm, atol=1e-5, rtol=1e-5) + + def testFrobeniusNormTens(self): + # Frobenius norm of a batch of TT-tensors. + with self.test_session() as sess: + tt = initializers.tensor_batch_with_random_cores((2, 1, 3), batch_size=3) + norm_sq_actual = ops.frobenius_norm_squared(tt) + norm_actual = ops.frobenius_norm(tt) + vars = [norm_sq_actual, norm_actual, ops.full(tt)] + norm_sq_actual_val, norm_actual_val, tt_val = sess.run(vars) + tt_val = tt_val.reshape((3, -1)) + norm_sq_desired_val = np.sum(tt_val * tt_val, axis=1) + norm_desired_val = np.sqrt(norm_sq_desired_val) + self.assertAllClose(norm_sq_actual_val, norm_sq_desired_val) + self.assertAllClose(norm_actual_val, norm_desired_val, atol=1e-5, + rtol=1e-5) + + def testMultiplyBatchByTensor(self): + tt_a = initializers.random_tensor((3, 3, 3), tt_rank=2) + tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) + with self.test_session() as sess: + res_actual = ops.full(ops.multiply(tt_a, tt_b)) + res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) + res_desired = ops.full(tt_a) * ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testMultiplyBatchByBatch(self): + tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) + tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) + res_actual = ops.full(ops.multiply(tt_a, tt_b)) + res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) + res_desired = ops.full(tt_a) * ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + with self.test_session() as sess: + res_actual = ops.full(ops.multiply(tt_a, tt_b)) + res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) + res_desired = ops.full(tt_a) * ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testMultiplyBroadcasting(self): + tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=1) + tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) + with self.test_session() as sess: + res_actual = ops.full(ops.multiply(tt_a, tt_b)) + res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) + res_desired = ops.full(tt_a) * ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testMultiplyUnknownBatchSizeBroadcasting(self): + c1 = tf.placeholder(tf.float32, [None, 1, 3, 2]) + c2 = tf.placeholder(tf.float32, [None, 2, 3, 1]) + tt_a = TensorTrainBatch([c1, c2]) + tt_b = initializers.random_tensor_batch((3, 3), tt_rank=3, batch_size=1) + tt_c = initializers.random_tensor((3, 3), tt_rank=3) + res_ab = ops.full(ops.multiply(tt_a, tt_b)) + res_ba = ops.full(ops.multiply(tt_b, tt_a)) + res_ac = ops.full(ops.multiply(tt_a, tt_c)) + res_ca = ops.full(ops.multiply(tt_c, tt_a)) + res_desired_ab = ops.full(tt_a) * ops.full(tt_b) + res_desired_ac = ops.full(tt_a) * ops.full(tt_c) + to_run = [res_ab, res_ba, res_ac, res_ca, res_desired_ab, res_desired_ac] + feed_dict = {c1:np.random.rand(7, 1, 3, 2), + c2:np.random.rand(7, 2, 3, 1)} + with self.test_session() as sess: + ab, ba, ac, ca, des_ab, des_ac = sess.run(to_run, feed_dict=feed_dict) + self.assertAllClose(ab, des_ab) + self.assertAllClose(ba, des_ab) + self.assertAllClose(ac, des_ac) + self.assertAllClose(ca, des_ac) + + def testMultiplyTwoBatchesUnknownSize(self): + c1 = tf.placeholder(tf.float32, [None, 1, 3, 2]) + c2 = tf.placeholder(tf.float32, [None, 2, 3, 1]) + c3 = tf.placeholder(tf.float32, [None, 1, 3, 2]) + c4 = tf.placeholder(tf.float32, [None, 2, 3, 1]) + tt_a = TensorTrainBatch([c1, c2]) + tt_b = TensorTrainBatch([c3, c4]) + res_ab = ops.full(ops.multiply(tt_a, tt_b)) + res_ba = ops.full(ops.multiply(tt_b, tt_a)) + res_desired = ops.full(tt_a) * ops.full(tt_b) + to_run = [res_ab, res_ba, res_desired] + feed_dict = {c1:np.random.rand(7, 1, 3, 2), + c2:np.random.rand(7, 2, 3, 1), + c3:np.random.rand(7, 1, 3, 2), + c4:np.random.rand(7, 2, 3, 1)} + + feed_dict_err = {c1:np.random.rand(7, 1, 3, 2), + c2:np.random.rand(7, 2, 3, 1), + c3:np.random.rand(1, 1, 3, 2), + c4:np.random.rand(1, 2, 3, 1)} + + with self.test_session() as sess: + ab_full, ba_full, des_full = sess.run(to_run, feed_dict=feed_dict) + self.assertAllClose(ab_full, des_full) + self.assertAllClose(ba_full, des_full) + with self.assertRaises(tf.errors.InvalidArgumentError): + sess.run(to_run, feed_dict=feed_dict_err) + + def testMultiplyUnknownSizeBatchAndBatch(self): + c1 = tf.placeholder(tf.float32, [None, 1, 3, 2]) + c2 = tf.placeholder(tf.float32, [None, 2, 3, 1]) + tt_b = initializers.random_tensor_batch((3, 3), tt_rank=2, batch_size=8) + tt_a = TensorTrainBatch([c1, c2]) + res_ab = ops.full(ops.multiply(tt_a, tt_b)) + res_ba = ops.full(ops.multiply(tt_b, tt_a)) + res_desired = ops.full(tt_a) * ops.full(tt_b) + to_run = [res_ab, res_ba, res_desired] + feed_dict = {c1:np.random.rand(8, 1, 3, 2), + c2:np.random.rand(8, 2, 3, 1)} + + feed_dict_err = {c1:np.random.rand(1, 1, 3, 2), + c2:np.random.rand(1, 2, 3, 1)} + + with self.test_session() as sess: + ab_full, ba_full, des_full = sess.run(to_run, feed_dict=feed_dict) + self.assertAllClose(ab_full, des_full) + self.assertAllClose(ba_full, des_full) + with self.assertRaises(tf.errors.InvalidArgumentError): + sess.run(to_run, feed_dict=feed_dict_err) + + def testGatherND(self): + idx = [[0, 0, 0], [0, 1, 2], [0, 1, 0]] + pl_idx = tf.placeholder(tf.int32, [None, 3]) + tt = initializers.random_tensor((3, 4, 5), tt_rank=2) + res_np = ops.gather_nd(tt, idx) + res_pl = ops.gather_nd(tt, pl_idx) + res_desired = tf.gather_nd(ops.full(tt), idx) + to_run = [res_np, res_pl, res_desired] + with self.test_session() as sess: + res_np_v, res_pl_v, des_v = sess.run(to_run, feed_dict={pl_idx: idx}) + self.assertAllClose(res_np_v, des_v) + self.assertAllClose(res_pl_v, res_pl_v) + + def testGatherNDBatch(self): + idx = [[0, 0, 0, 0], [1, 0, 1, 2], [0, 0, 1, 0]] + pl_idx = tf.placeholder(tf.int32, [None, 4]) + tt = initializers.random_tensor_batch((3, 4, 5), tt_rank=2, batch_size=2) + res_np = ops.gather_nd(tt, idx) + res_pl = ops.gather_nd(tt, pl_idx) + res_desired = tf.gather_nd(ops.full(tt), idx) + to_run = [res_np, res_pl, res_desired] + with self.test_session() as sess: + res_np_v, res_pl_v, des_v = sess.run(to_run, feed_dict={pl_idx: idx}) + self.assertAllClose(res_np_v, des_v) + self.assertAllClose(res_pl_v, res_pl_v) + + def testCoreRenormBatch(self): + a = initializers.random_tensor_batch(3 * (10,), tt_rank=7, batch_size=5) + b = ops.renormalize_tt_cores(a) + var_list = [ops.full(a), ops.full(b)] + + with self.test_session() as sess: + af, bf = sess.run(var_list) + b_cores = sess.run(b.tt_cores) + b_cores_norms = [] + for cr in b_cores: + b_cores_norms.append(np.linalg.norm(cr)) + self.assertAllClose(af, bf, atol=1e-5, rtol=1e-5) + self.assertAllClose(b_cores_norms, b_cores_norms[0] + * np.ones((len(b_cores)))) + +class TTMatrixTestBatch(tf.test.TestCase): + + def testFullMatrix2d(self): + np.random.seed(1) + for rank in [1, 2]: + a = np.random.rand(3, 2, 3, rank).astype(np.float32) + b = np.random.rand(3, rank, 4, 5).astype(np.float32) + tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape((3, rank, 4, 5, 1))) + # Basically do full by hand. + desired = np.einsum('oijb,obkl->oijkl', a, b) + desired = desired.reshape((3, 2, 3, 4, 5)) + desired = desired.transpose((0, 1, 3, 2, 4)) + desired = desired.reshape((3, 2 * 4, 3 * 5)) + with self.test_session(): + tf_mat = TensorTrainBatch(tt_cores) + actual = ops.full(tf_mat) + self.assertAllClose(desired, actual.eval()) + + def testFullMatrix3d(self): + np.random.seed(1) + for rank in [1, 2]: + a = np.random.rand(3, 2, 3, rank).astype(np.float32) + b = np.random.rand(3, rank, 4, 5, rank).astype(np.float32) + c = np.random.rand(3, rank, 2, 2).astype(np.float32) + tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape(3, rank, 4, 5, rank), + c.reshape(3, rank, 2, 2, 1)) + # Basically do full by hand. + desired = np.einsum('oija,oaklb,obpq->oijklpq', a, b, c) + desired = desired.reshape((3, 2, 3, 4, 5, 2, 2)) + desired = desired.transpose((0, 1, 3, 5, 2, 4, 6)) + desired = desired.reshape((3, 2 * 4 * 2, 3 * 5 * 2)) + with self.test_session(): + tf_mat = TensorTrainBatch(tt_cores) + actual = ops.full(tf_mat) + self.assertAllClose(desired, actual.eval()) + + def testTTMatTimesTTMatSameBatchSize(self): + # Multiply a batch of TT-matrices by another batch of TT-matrices with the + # same batch sizes. + left_shape = (2, 3) + sum_shape = (4, 3) + right_shape = (4, 4) + with self.test_session() as sess: + tt_mat_1 = initializers.random_matrix_batch((left_shape, sum_shape), + tt_rank=3, batch_size=3) + tt_mat_2 = initializers.random_matrix_batch((sum_shape, right_shape), + batch_size=3) + res_actual = ops.matmul(tt_mat_1, tt_mat_2) + res_actual = ops.full(res_actual) + res_desired = tf.matmul(ops.full(tt_mat_1), ops.full(tt_mat_2)) + res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) + # TODO: why so bad accuracy? + self.assertAllClose(res_actual_val, res_desired_val, atol=1e-5, rtol=1e-5) + + def testTTMatTimesTTMatBroadcasting(self): + # Multiply a batch of TT-matrices by another batch of TT-matrices with + # broadcasting. + left_shape = (2, 3) + sum_shape = (4, 3) + right_shape = (4, 4) + with self.test_session() as sess: + tt_mat_1 = initializers.random_matrix_batch((left_shape, sum_shape), + tt_rank=3, batch_size=3) + tt_mat_2 = initializers.random_matrix_batch((sum_shape, right_shape)) + # TT-batch by one element TT-batch + res_actual = ops.matmul(tt_mat_1, tt_mat_2) + res_actual = ops.full(res_actual) + # TT by TT-batch. + res_actual2 = ops.matmul(ops.transpose(tt_mat_2[0]), ops.transpose(tt_mat_1)) + res_actual2 = ops.full(ops.transpose(res_actual2)) + res_desired = tf.einsum('oij,jk->oik', ops.full(tt_mat_1), + ops.full(tt_mat_2[0])) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val, atol=1e-5, rtol=1e-5) + self.assertAllClose(res_actual2_val, res_desired_val, atol=1e-5, + rtol=1e-5) + + def testTranspose(self): + # Transpose a batch of TT-matrices. + with self.test_session() as sess: + tt = initializers.random_matrix_batch(((2, 3, 4), (2, 2, 2)), batch_size=2) + res_actual = ops.full(ops.transpose(tt)) + res_actual_val, tt_val = sess.run([res_actual, ops.full(tt)]) + self.assertAllClose(tt_val.transpose((0, 2, 1)), res_actual_val) + + def testAddSameBatchSize(self): + # Sum two TT-matrices with the same batch size. + tt_a = initializers.random_matrix_batch(((2, 1, 4), None), tt_rank=2, + batch_size=3) + tt_b = initializers.random_matrix_batch(((2, 1, 4), None), + tt_rank=[1, 2, 4, 1], batch_size=3) + with self.test_session() as sess: + res_actual = ops.full(ops.add(tt_a, tt_b)) + res_actual2 = ops.full(tt_a + tt_b) + res_desired = ops.full(tt_a) + ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testAddBroadcasting(self): + # Sum two TT-matrices with broadcasting. + tt_a = initializers.random_matrix_batch(((2, 1, 4), (2, 2, 2)), tt_rank=2, + batch_size=3) + tt_b = initializers.random_matrix_batch(((2, 1, 4), (2, 2, 2)), + tt_rank=[1, 2, 4, 1], batch_size=1) + with self.test_session() as sess: + res_actual = ops.full(ops.add(tt_a, tt_b)) + res_actual2 = ops.full(tt_b + tt_a) + res_desired = ops.full(tt_a) + ops.full(tt_b) + to_run = [res_actual, res_actual2, res_desired] + res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) + self.assertAllClose(res_actual_val, res_desired_val) + self.assertAllClose(res_actual2_val, res_desired_val) + + def testCastFloat(self): + # Test cast function for float tt-matrices and vectors. + tt_mat = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=2, + batch_size=3) + + with self.test_session() as sess: + for dtype in [tf.float16, tf.float32, tf.float64]: + casted = ops.cast(tt_mat, dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(dtype, casted.dtype) + self.assertTrue(dtype, casted_val.dtype) + + def testCastIntFloat(self): + # Tests cast function from int to float for matrices. + np.random.seed(1) + K_1 = np.random.randint(0, high=100, size=(1, 2, 2, 2)) + K_2 = np.random.randint(0, high=100, size=(2, 3, 3, 2)) + K_3 = np.random.randint(0, high=100, size=(2, 2, 2, 1)) + tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) + tt_int_batch = shapes.expand_batch_dim(tt_int) + + with self.test_session() as sess: + for dtype in [tf.float16, tf.float32, tf.float64]: + casted = ops.cast(tt_int_batch, dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(dtype, casted.dtype) + self.assertTrue(dtype, casted_val.dtype) + + +def _random_sparse(shape, non_zeros): + sparse_flat_indices = np.random.choice(np.prod(shape), non_zeros).astype(int) + sparse_indices = np.unravel_index(sparse_flat_indices, shape) + sparse_indices = np.vstack(sparse_indices).transpose() + values = np.random.randn(non_zeros).astype(np.float32) + sparse = tf.SparseTensor(indices=sparse_indices, values=values, + dense_shape=shape) + return sparse + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/regularizers.py b/build/lib/t3f/regularizers.py new file mode 100644 index 00000000..6928e329 --- /dev/null +++ b/build/lib/t3f/regularizers.py @@ -0,0 +1,79 @@ +import numbers + +import tensorflow as tf + +from t3f import ops + + +def l2_regularizer(scale, scope=None): + """Returns a function that applies L2 regularization to TensorTrain weights. + + Args: + scale: A scalar multiplier `Tensor`. 0.0 disables the regularizer. + scope: An optional scope name. + + Returns: + A function with signature `l2(tt)` that applies L2 regularization. + + Raises: + ValueError: If scale is negative or if scale is not a float. + """ + if isinstance(scale, numbers.Integral): + raise ValueError('scale cannot be an integer: %s' % (scale,)) + if isinstance(scale, numbers.Real): + if scale < 0.: + raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % + scale) + if scale == 0.: + tf.logging.info('Scale of 0 disables regularizer.') + return lambda _: None + + def l2(tt): + """Applies l2 regularization to TensorTrain object.""" + with tf.name_scope(scope, 'l2_regularizer', [tt]) as name: + my_scale = tf.convert_to_tensor(scale, + dtype=tt.dtype.base_dtype, + name='scale') + return tf.mul(my_scale, ops.frobenius_norm_squared(tt), name=name) + + return l2 + + +def cores_regularizer(core_regularizer, scale, scope=None): + """Returns a function that applies given regularization to each TT-core. + + Args: + core_regularizer: a function with signature `core_regularizer(core)` that + returns the penalty for the given TT-core. + scale: A scalar multiplier `Tensor`. 0.0 disables the regularizer. + scope: An optional scope name. + + Returns: + A function with signature `regularizer(weights)` that applies + the regularization. + + Raises: + ValueError: If scale is negative or if scale is not a float. + """ + if isinstance(scale, numbers.Integral): + raise ValueError('scale cannot be an integer: %s' % (scale,)) + if isinstance(scale, numbers.Real): + if scale < 0.: + raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % + scale) + if scale == 0.: + tf.logging.info('Scale of 0 disables regularizer.') + return lambda _: None + + def regularizer(tt): + """Applies the regularization to TensorTrain object.""" + with tf.name_scope(scope, 'l2_regularizer', [tt]) as name: + my_scale = tf.convert_to_tensor(scale, + dtype=tt.dtype.base_dtype, + name='scale') + penalty = 0.0 + for i in range(tt.ndims()): + penalty += core_regularizer(tt.tt_cores[i]) + return tf.mul(my_scale, penalty, name=name) + + return regularizer diff --git a/build/lib/t3f/riemannian.py b/build/lib/t3f/riemannian.py new file mode 100644 index 00000000..21a49748 --- /dev/null +++ b/build/lib/t3f/riemannian.py @@ -0,0 +1,751 @@ +import tensorflow as tf + +from t3f.tensor_train import TensorTrain +from t3f.tensor_train_batch import TensorTrainBatch +from t3f import shapes +from t3f import decompositions + + +def project_sum(what, where, weights=None): + """Project sum of `what` TTs on the tangent space of `where` TT. + + project_sum(what, x) = P_x(what) + project_sum(batch_what, x) = P_x(\sum_i batch_what[i]) + project_sum(batch_what, x, weights) = P_x(\sum_j weights[j] * batch_what[j]) + + This function implements the algorithm from the paper [1], theorem 3.1. + + [1] C. Lubich, I. Oseledets and B. Vandereycken, Time integration of + Tensor Trains. + + Args: + what: TensorTrain or TensorTrainBatch. In the case of batch returns + projection of the sum of elements in the batch. + where: TensorTrain, TT-tensor or TT-matrix on which tangent space to project + weights: python list or tf.Tensor of numbers or None, weights of the sum + + Returns: + a TensorTrain with the TT-ranks equal 2 * tangent_space_tens.get_tt_ranks() + + Complexity: + O(d r_where^3 m) for orthogonalizing the TT-cores of where + +O(batch_size d r_what r_where n (r_what + r_where)) + d is the number of TT-cores (what.ndims()); + r_what is the largest TT-rank of what max(what.get_tt_rank()) + r_where is the largest TT-rank of where + n is the size of the axis dimension of what and where e.g. + for a tensor of size 4 x 4 x 4, n is 4; + for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 + """ + # Always work with batch of TT objects for simplicity. + what = shapes.expand_batch_dim(what) + + if weights is not None: + weights = tf.convert_to_tensor(weights) + + if not isinstance(where, TensorTrain): + raise ValueError('The first argument should be a TensorTrain object, got ' + '"%s".' % where) + + if where.get_raw_shape() != what.get_raw_shape(): + raise ValueError('The shapes of the tensor we want to project and of the ' + 'tensor on which tangent space we want to project should ' + 'match, got %s and %s.' % + (where.get_raw_shape(), + what.get_raw_shape())) + + dtypes_compatible = (where.dtype.is_compatible_with(what.dtype) or + what.dtype.is_compatible_with(where.dtype)) + if not dtypes_compatible: + raise ValueError('Dtypes of the arguments should coincide, got %s and %s.' % + (where.dtype, + what.dtype)) + + left_tangent_space_tens = decompositions.orthogonalize_tt_cores( + where) + right_tangent_space_tens = decompositions.orthogonalize_tt_cores( + left_tangent_space_tens, left_to_right=False) + + ndims = where.ndims() + dtype = where.dtype + raw_shape = shapes.lazy_raw_shape(where) + batch_size = shapes.lazy_batch_size(what) + right_tangent_tt_ranks = shapes.lazy_tt_ranks(right_tangent_space_tens) + left_tangent_tt_ranks = shapes.lazy_tt_ranks(left_tangent_space_tens) + + # For einsum notation. + mode_str = 'ij' if where.is_tt_matrix() else 'i' + right_rank_dim = where.right_tt_rank_dim + left_rank_dim = where.left_tt_rank_dim + if weights is not None: + weights_shape = weights.get_shape() + output_is_batch = len(weights_shape) > 1 and weights_shape[1] > 1 + else: + output_is_batch = False + output_batch_str = 'o' if output_is_batch else '' + if output_is_batch: + right_rank_dim += 1 + left_rank_dim += 1 + output_batch_size = weights.get_shape()[1].value + + # Prepare rhs vectors. + # rhs[core_idx] is of size + # batch_size x tensor_tt_ranks[core_idx] x tangent_tt_ranks[core_idx] + rhs = [None] * (ndims + 1) + rhs[ndims] = tf.ones((batch_size, 1, 1), dtype=dtype) + for core_idx in range(ndims - 1, 0, -1): + tens_core = what.tt_cores[core_idx] + right_tang_core = right_tangent_space_tens.tt_cores[core_idx] + einsum_str = 'sa{0}b,sbd,c{0}d->sac'.format(mode_str) + rhs[core_idx] = tf.einsum(einsum_str, tens_core, rhs[core_idx + 1], + right_tang_core) + + # Prepare lhs vectors. + # lhs[core_idx] is of size + # batch_size x tangent_tt_ranks[core_idx] x tensor_tt_ranks[core_idx] + lhs = [None] * (ndims + 1) + lhs[0] = tf.ones((batch_size, 1, 1), dtype=dtype) + for core_idx in range(ndims - 1): + tens_core = what.tt_cores[core_idx] + left_tang_core = left_tangent_space_tens.tt_cores[core_idx] + einsum_str = 'sab,a{0}c,sb{0}d->scd'.format(mode_str) + lhs[core_idx + 1] = tf.einsum(einsum_str, lhs[core_idx], left_tang_core, + tens_core) + + # Left to right sweep. + res_cores_list = [] + for core_idx in range(ndims): + tens_core = what.tt_cores[core_idx] + left_tang_core = left_tangent_space_tens.tt_cores[core_idx] + right_tang_core = right_tangent_space_tens.tt_cores[core_idx] + + if core_idx < ndims - 1: + einsum_str = 'sab,sb{0}c->sa{0}c'.format(mode_str) + proj_core = tf.einsum(einsum_str, lhs[core_idx], tens_core) + einsum_str = 'a{0}b,sbc->sa{0}c'.format(mode_str) + proj_core -= tf.einsum(einsum_str, left_tang_core, lhs[core_idx + 1]) + if weights is None: + einsum_str = 'sa{0}b,sbc->a{0}c'.format(mode_str) + proj_core = tf.einsum(einsum_str, proj_core, rhs[core_idx + 1]) + else: + einsum_str = 'sa{0}b,sbc->sa{0}c'.format(mode_str, output_batch_str) + proj_core_s = tf.einsum(einsum_str, proj_core, rhs[core_idx + 1]) + einsum_str = 's{1},sa{0}c->{1}a{0}c'.format(mode_str, output_batch_str) + proj_core = tf.einsum(einsum_str, weights, proj_core_s) + + if core_idx == ndims - 1: + if weights is None: + einsum_str = 'sab,sb{0}c->a{0}c'.format(mode_str) + proj_core = tf.einsum(einsum_str, lhs[core_idx], tens_core) + else: + einsum_str = 'sab,sb{0}c->sa{0}c'.format(mode_str, output_batch_str) + proj_core_s = tf.einsum(einsum_str, lhs[core_idx], tens_core) + einsum_str = 's{1},sa{0}c->{1}a{0}c'.format(mode_str, output_batch_str) + proj_core = tf.einsum(einsum_str, weights, proj_core_s) + + if output_is_batch: + # Add batch dimension of size output_batch_size to left_tang_core and + # right_tang_core + extended_left_tang_core = tf.expand_dims(left_tang_core, 0) + extended_right_tang_core = tf.expand_dims(right_tang_core, 0) + if where.is_tt_matrix(): + extended_left_tang_core = tf.tile(extended_left_tang_core, + [output_batch_size, 1, 1, 1, 1]) + extended_right_tang_core = tf.tile(extended_right_tang_core, + [output_batch_size, 1, 1, 1, 1]) + else: + extended_left_tang_core = tf.tile(extended_left_tang_core, + [output_batch_size, 1, 1, 1]) + extended_right_tang_core = tf.tile(extended_right_tang_core, + [output_batch_size, 1, 1, 1]) + else: + extended_left_tang_core = left_tang_core + extended_right_tang_core = right_tang_core + + if core_idx == 0: + res_core = tf.concat((proj_core, extended_left_tang_core), + axis=right_rank_dim) + elif core_idx == ndims - 1: + res_core = tf.concat((extended_right_tang_core, proj_core), axis=left_rank_dim) + else: + rank_1 = right_tangent_tt_ranks[core_idx] + rank_2 = left_tangent_tt_ranks[core_idx + 1] + if where.is_tt_matrix(): + mode_size_n = raw_shape[0][core_idx] + mode_size_m = raw_shape[1][core_idx] + shape = [rank_1, mode_size_n, mode_size_m, rank_2] + else: + mode_size = raw_shape[0][core_idx] + shape = [rank_1, mode_size, rank_2] + if output_is_batch: + shape = [output_batch_size] + shape + zeros = tf.zeros(shape, dtype) + upper = tf.concat((extended_right_tang_core, zeros), axis=right_rank_dim) + lower = tf.concat((proj_core, extended_left_tang_core), + axis=right_rank_dim) + res_core = tf.concat((upper, lower), axis=left_rank_dim) + res_cores_list.append(res_core) + # TODO: TT-ranks. + if output_is_batch: + res = TensorTrainBatch(res_cores_list, where.get_raw_shape(), + batch_size=output_batch_size) + else: + res = TensorTrain(res_cores_list, where.get_raw_shape()) + + res.projection_on = where + return res + + +def project(what, where): + """Project `what` TTs on the tangent space of `where` TT. + + project(what, x) = P_x(what) + project(batch_what, x) = batch(P_x(batch_what[0]), ..., P_x(batch_what[N])) + + This function implements the algorithm from the paper [1], theorem 3.1. + + [1] C. Lubich, I. Oseledets and B. Vandereycken, Time integration of + Tensor Trains. + + Args: + what: TensorTrain or TensorTrainBatch. In the case of batch returns + batch with projection of each individual tensor. + where: TensorTrain, TT-tensor or TT-matrix on which tangent space to project + + Returns: + a TensorTrain with the TT-ranks equal 2 * tangent_space_tens.get_tt_ranks() + + Complexity: + O(d r_where^3 m) for orthogonalizing the TT-cores of where + +O(batch_size d r_what r_where n (r_what + r_where)) + d is the number of TT-cores (what.ndims()); + r_what is the largest TT-rank of what max(what.get_tt_rank()) + r_where is the largest TT-rank of where + n is the size of the axis dimension of what and where e.g. + for a tensor of size 4 x 4 x 4, n is 4; + for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 + """ + + if not isinstance(where, TensorTrain): + raise ValueError('The first argument should be a TensorTrain object, got ' + '"%s".' % where) + + if where.get_raw_shape() != what.get_raw_shape(): + raise ValueError('The shapes of the tensor we want to project and of the ' + 'tensor on which tangent space we want to project should ' + 'match, got %s and %s.' % + (where.get_raw_shape(), + what.get_raw_shape())) + dtypes_compatible = (where.dtype.is_compatible_with(what.dtype) or + what.dtype.is_compatible_with(where.dtype)) + if not dtypes_compatible: + raise ValueError('Dtypes of the arguments should coincide, got %s and %s.' % + (where.dtype, + what.dtype)) + + left_tangent_space_tens = decompositions.orthogonalize_tt_cores( + where) + right_tangent_space_tens = decompositions.orthogonalize_tt_cores( + left_tangent_space_tens, left_to_right=False) + + ndims = where.ndims() + dtype = where.dtype + raw_shape = shapes.lazy_raw_shape(where) + right_tangent_tt_ranks = shapes.lazy_tt_ranks(right_tangent_space_tens) + left_tangent_tt_ranks = shapes.lazy_tt_ranks(left_tangent_space_tens) + + # For einsum notation. + mode_str = 'ij' if where.is_tt_matrix() else 'i' + right_rank_dim = what.right_tt_rank_dim + left_rank_dim = what.left_tt_rank_dim + output_is_batch = isinstance(what, TensorTrainBatch) + if output_is_batch: + output_batch_size = what.batch_size + + # Always work with batch of TT objects for simplicity. + what = shapes.expand_batch_dim(what) + batch_size = shapes.lazy_batch_size(what) + + # Prepare rhs vectors. + # rhs[core_idx] is of size + # batch_size x tensor_tt_ranks[core_idx] x tangent_tt_ranks[core_idx] + rhs = [None] * (ndims + 1) + rhs[ndims] = tf.ones((batch_size, 1, 1), dtype=dtype) + for core_idx in range(ndims - 1, 0, -1): + tens_core = what.tt_cores[core_idx] + right_tang_core = right_tangent_space_tens.tt_cores[core_idx] + einsum_str = 'sa{0}b,sbd,c{0}d->sac'.format(mode_str) + rhs[core_idx] = tf.einsum(einsum_str, tens_core, rhs[core_idx + 1], + right_tang_core) + + # Prepare lhs vectors. + # lhs[core_idx] is of size + # batch_size x tangent_tt_ranks[core_idx] x tensor_tt_ranks[core_idx] + lhs = [None] * (ndims + 1) + lhs[0] = tf.ones((batch_size, 1, 1), dtype=dtype) + for core_idx in range(ndims - 1): + tens_core = what.tt_cores[core_idx] + left_tang_core = left_tangent_space_tens.tt_cores[core_idx] + einsum_str = 'sab,a{0}c,sb{0}d->scd'.format(mode_str) + lhs[core_idx + 1] = tf.einsum(einsum_str, lhs[core_idx], left_tang_core, + tens_core) + + # Left to right sweep. + res_cores_list = [] + for core_idx in range(ndims): + tens_core = what.tt_cores[core_idx] + left_tang_core = left_tangent_space_tens.tt_cores[core_idx] + right_tang_core = right_tangent_space_tens.tt_cores[core_idx] + + if core_idx < ndims - 1: + einsum_str = 'sab,sb{0}c->sa{0}c'.format(mode_str) + proj_core = tf.einsum(einsum_str, lhs[core_idx], tens_core) + einsum_str = 'a{0}b,sbc->sa{0}c'.format(mode_str) + proj_core -= tf.einsum(einsum_str, left_tang_core, lhs[core_idx + 1]) + if output_is_batch: + einsum_str = 'sa{0}b,sbc->sa{0}c'.format(mode_str) + else: + einsum_str = 'sa{0}b,sbc->a{0}c'.format(mode_str) + proj_core = tf.einsum(einsum_str, proj_core, rhs[core_idx + 1]) + + if core_idx == ndims - 1: + if output_is_batch: + einsum_str = 'sab,sb{0}c->sa{0}c'.format(mode_str) + else: + einsum_str = 'sab,sb{0}c->a{0}c'.format(mode_str) + proj_core = tf.einsum(einsum_str, lhs[core_idx], tens_core) + + if output_is_batch: + # Add batch dimension of size output_batch_size to left_tang_core and + # right_tang_core + extended_left_tang_core = tf.expand_dims(left_tang_core, 0) + extended_right_tang_core = tf.expand_dims(right_tang_core, 0) + if where.is_tt_matrix(): + extended_left_tang_core = tf.tile(extended_left_tang_core, + [output_batch_size, 1, 1, 1, 1]) + extended_right_tang_core = tf.tile(extended_right_tang_core, + [output_batch_size, 1, 1, 1, 1]) + else: + extended_left_tang_core = tf.tile(extended_left_tang_core, + [output_batch_size, 1, 1, 1]) + extended_right_tang_core = tf.tile(extended_right_tang_core, + [output_batch_size, 1, 1, 1]) + else: + extended_left_tang_core = left_tang_core + extended_right_tang_core = right_tang_core + + if core_idx == 0: + res_core = tf.concat((proj_core, extended_left_tang_core), + axis=right_rank_dim) + elif core_idx == ndims - 1: + res_core = tf.concat((extended_right_tang_core, proj_core), axis=left_rank_dim) + else: + rank_1 = right_tangent_tt_ranks[core_idx] + rank_2 = left_tangent_tt_ranks[core_idx + 1] + if where.is_tt_matrix(): + mode_size_n = raw_shape[0][core_idx] + mode_size_m = raw_shape[1][core_idx] + shape = [rank_1, mode_size_n, mode_size_m, rank_2] + else: + mode_size = raw_shape[0][core_idx] + shape = [rank_1, mode_size, rank_2] + if output_is_batch: + shape = [output_batch_size] + shape + zeros = tf.zeros(shape, dtype) + upper = tf.concat((extended_right_tang_core, zeros), axis=right_rank_dim) + lower = tf.concat((proj_core, extended_left_tang_core), + axis=right_rank_dim) + res_core = tf.concat((upper, lower), axis=left_rank_dim) + res_cores_list.append(res_core) + # TODO: TT-ranks. + if output_is_batch: + res = TensorTrainBatch(res_cores_list, where.get_raw_shape(), + batch_size=output_batch_size) + else: + res = TensorTrain(res_cores_list, where.get_raw_shape()) + + res.projection_on = where + return res + + +def project_matmul(what, where, matrix): + """Project `matrix` * `what` TTs on the tangent space of `where` TT. + + project(what, x) = P_x(what) + project(batch_what, x) = batch(P_x(batch_what[0]), ..., P_x(batch_what[N])) + + This function implements the algorithm from the paper [1], theorem 3.1. + + [1] C. Lubich, I. Oseledets and B. Vandereycken, Time integration of + Tensor Trains. + + Args: + what: TensorTrain or TensorTrainBatch. In the case of batch returns + batch with projection of each individual tensor. + where: TensorTrain, TT-tensor or TT-matrix on which tangent space to project + matrix: TensorTrain, TT-matrix to multiply by what + + Returns: + a TensorTrain with the TT-ranks equal 2 * tangent_space_tens.get_tt_ranks() + + Complexity: + O(d r_where^3 m) for orthogonalizing the TT-cores of where + +O(batch_size d R r_what r_where (n r_what + n m R + m r_where)) + d is the number of TT-cores (what.ndims()); + r_what is the largest TT-rank of what max(what.get_tt_rank()) + r_where is the largest TT-rank of where + matrix is of TT-rank R and of raw-shape (m, m, ..., m) x (n, n, ..., n). + """ + + if not isinstance(where, TensorTrain): + raise ValueError('The first argument should be a TensorTrain object, got ' + '"%s".' % where) + + if where.get_raw_shape() != what.get_raw_shape(): + raise ValueError('The shapes of the tensor we want to project and of the ' + 'tensor on which tangent space we want to project should ' + 'match, got %s and %s.' % + (where.get_raw_shape(), + what.get_raw_shape())) + + dtypes_compatible = (where.dtype.is_compatible_with(what.dtype) or + what.dtype.is_compatible_with(where.dtype)) + if not dtypes_compatible: + raise ValueError('Dtypes of the arguments should coincide, got %s and %s.' % + (where.dtype, + what.dtype)) + + left_tangent_space_tens = decompositions.orthogonalize_tt_cores( + where) + right_tangent_space_tens = decompositions.orthogonalize_tt_cores( + left_tangent_space_tens, left_to_right=False) + + ndims = where.ndims() + dtype = where.dtype + raw_shape = shapes.lazy_raw_shape(where) + batch_size = shapes.lazy_batch_size(what) + right_tangent_tt_ranks = shapes.lazy_tt_ranks(right_tangent_space_tens) + left_tangent_tt_ranks = shapes.lazy_tt_ranks(left_tangent_space_tens) + + # For einsum notation. + right_rank_dim = what.right_tt_rank_dim + left_rank_dim = what.left_tt_rank_dim + output_is_batch = isinstance(what, TensorTrainBatch) + if output_is_batch: + output_batch_size = what.batch_size + + # Always work with batch of TT objects for simplicity. + what = shapes.expand_batch_dim(what) + + # Prepare rhs vectors. + # rhs[core_idx] is of size + # batch_size x tensor_tt_ranks[core_idx] x matrix_tt_ranks[core_idx] x tangent_tt_ranks[core_idx] + rhs = [None] * (ndims + 1) + rhs[ndims] = tf.ones((batch_size, 1, 1, 1), dtype=dtype) + for core_idx in range(ndims - 1, 0, -1): + tens_core = what.tt_cores[core_idx] + right_tang_core = right_tangent_space_tens.tt_cores[core_idx] + matrix_core = matrix.tt_cores[core_idx] + rhs[core_idx] = tf.einsum('bije,cikf,sdef,sajkd->sabc', matrix_core, + right_tang_core, rhs[core_idx + 1], tens_core) + # Prepare lhs vectors. + # lhs[core_idx] is of size + # batch_size x tangent_tt_ranks[core_idx] x matrix_tt_ranks[core_idx] x tensor_tt_ranks[core_idx] + lhs = [None] * (ndims + 1) + lhs[0] = tf.ones((batch_size, 1, 1, 1), dtype=dtype) + for core_idx in range(ndims - 1): + tens_core = what.tt_cores[core_idx] + left_tang_core = left_tangent_space_tens.tt_cores[core_idx] + matrix_core = matrix.tt_cores[core_idx] + # TODO: brutforce order of indices in lhs?? + lhs[core_idx + 1] = tf.einsum('bije,aikd,sabc,scjkf->sdef', matrix_core, + left_tang_core, lhs[core_idx], tens_core) + + # Left to right sweep. + res_cores_list = [] + for core_idx in range(ndims): + tens_core = what.tt_cores[core_idx] + matrix_core = matrix.tt_cores[core_idx] + left_tang_core = left_tangent_space_tens.tt_cores[core_idx] + right_tang_core = right_tangent_space_tens.tt_cores[core_idx] + + if core_idx < ndims - 1: + proj_core = tf.einsum('scjke,sabc,bijd->saikde', tens_core, + lhs[core_idx], matrix_core) + proj_core -= tf.einsum('aikb,sbcd->saikcd', left_tang_core, + lhs[core_idx + 1]) + proj_core = tf.einsum('saikcb,sbcd->saikd', proj_core, rhs[core_idx + 1]) + + if core_idx == ndims - 1: + # d and e dimensions take 1 value, since its the last rank. + # To make the result shape (?, ?, ?, 1), we are summing d and leaving e, + # but we could have done the opposite -- sum e and leave d. + proj_core = tf.einsum('sabc,bijd,scjke->saike', lhs[core_idx], matrix_core, + tens_core) + + if output_is_batch: + # Add batch dimension of size output_batch_size to left_tang_core and + # right_tang_core + extended_left_tang_core = tf.expand_dims(left_tang_core, 0) + extended_right_tang_core = tf.expand_dims(right_tang_core, 0) + extended_left_tang_core = tf.tile(extended_left_tang_core, + [output_batch_size, 1, 1, 1, 1]) + extended_right_tang_core = tf.tile(extended_right_tang_core, + [output_batch_size, 1, 1, 1, 1]) + else: + extended_left_tang_core = left_tang_core + extended_right_tang_core = right_tang_core + + if core_idx == 0: + res_core = tf.concat((proj_core, extended_left_tang_core), + axis=right_rank_dim) + elif core_idx == ndims - 1: + res_core = tf.concat((extended_right_tang_core, proj_core), + axis=left_rank_dim) + else: + rank_1 = right_tangent_tt_ranks[core_idx] + rank_2 = left_tangent_tt_ranks[core_idx + 1] + mode_size_n = raw_shape[0][core_idx] + mode_size_m = raw_shape[1][core_idx] + shape = [rank_1, mode_size_n, mode_size_m, rank_2] + if output_is_batch: + shape = [output_batch_size] + shape + zeros = tf.zeros(shape, dtype) + upper = tf.concat((extended_right_tang_core, zeros), + axis=right_rank_dim) + lower = tf.concat((proj_core, extended_left_tang_core), + axis=right_rank_dim) + res_core = tf.concat((upper, lower), axis=left_rank_dim) + res_cores_list.append(res_core) + + # TODO: TT-ranks. + if output_is_batch: + res = TensorTrainBatch(res_cores_list, where.get_raw_shape(), + batch_size=output_batch_size) + else: + res = TensorTrain(res_cores_list, where.get_raw_shape()) + + res.projection_on = where + return res + + +def pairwise_flat_inner_projected(projected_tt_vectors_1, + projected_tt_vectors_2): + """Scalar products between two batches of TTs from the same tangent space. + + res[i, j] = t3f.flat_inner(projected_tt_vectors_1[i], projected_tt_vectors_1[j]). + + pairwise_flat_inner_projected(projected_tt_vectors_1, projected_tt_vectors_2) + is equivalent to + pairwise_flat_inner(projected_tt_vectors_1, projected_tt_vectors_2) + , but works only on objects from the same tangent space and is much faster + than general pairwise_flat_inner. + + Args: + projected_tt_vectors_1: TensorTrainBatch of tensors projected on the same + tangent space as projected_tt_vectors_2. + projected_tt_vectors_2: TensorTrainBatch. + + Returns: + tf.tensor with the scalar product matrix. + + Complexity: + O(batch_size^2 d r^2 n), where + d is the number of TT-cores (projected_tt_vectors_1.ndims()); + r is the largest TT-rank max(projected_tt_vectors_1.get_tt_rank()) + (i.e. 2 * {the TT-rank of the object we projected vectors onto}. + and n is the size of the axis dimension, e.g. + for a tensor of size 4 x 4 x 4, n is 4; + for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12. + """ + if not hasattr(projected_tt_vectors_1, 'projection_on') or \ + not hasattr(projected_tt_vectors_2, 'projection_on'): + raise ValueError('Both arguments should be projections on the tangent ' + 'space of some other TT-object. All projection* functions ' + 'leave .projection_on field in the resulting TT-object ' + 'which is not present in the arguments you\'ve provided') + + if projected_tt_vectors_1.projection_on != projected_tt_vectors_2.projection_on: + raise ValueError('Both arguments should be projections on the tangent ' + 'space of the same TT-object. The provided arguments are ' + 'projections on different TT-objects (%s and %s). Or at ' + 'least the pointers are different.' % + (projected_tt_vectors_1.projection_on, + projected_tt_vectors_2.projection_on)) + + # Always work with batches of objects for simplicity. + projected_tt_vectors_1 = shapes.expand_batch_dim(projected_tt_vectors_1) + projected_tt_vectors_2 = shapes.expand_batch_dim(projected_tt_vectors_2) + + ndims = projected_tt_vectors_1.ndims() + tt_ranks = shapes.lazy_tt_ranks(projected_tt_vectors_1) + + if projected_tt_vectors_1.is_tt_matrix(): + right_size = tt_ranks[1] // 2 + curr_core_1 = projected_tt_vectors_1.tt_cores[0] + curr_core_2 = projected_tt_vectors_2.tt_cores[0] + curr_du_1 = curr_core_1[:, :, :, :, :right_size] + curr_du_2 = curr_core_2[:, :, :, :, :right_size] + res = tf.einsum('paijb,qaijb->pq', curr_du_1, curr_du_2) + for core_idx in range(1, ndims): + left_size = tt_ranks[core_idx] // 2 + right_size = tt_ranks[core_idx + 1] // 2 + curr_core_1 = projected_tt_vectors_1.tt_cores[core_idx] + curr_core_2 = projected_tt_vectors_2.tt_cores[core_idx] + curr_du_1 = curr_core_1[:, left_size:, :, :, :right_size] + curr_du_2 = curr_core_2[:, left_size:, :, :, :right_size] + res += tf.einsum('paijb,qaijb->pq', curr_du_1, curr_du_2) + + left_size = tt_ranks[-2] // 2 + curr_core_1 = projected_tt_vectors_1.tt_cores[-1] + curr_core_2 = projected_tt_vectors_2.tt_cores[-1] + curr_du_1 = curr_core_1[:, left_size:, :, :, :] + curr_du_2 = curr_core_2[:, left_size:, :, :, :] + res += tf.einsum('paijb,qaijb->pq', curr_du_1, curr_du_2) + else: + # Working with TT-tensor, not TT-matrix. + right_size = tt_ranks[1] // 2 + curr_core_1 = projected_tt_vectors_1.tt_cores[0] + curr_core_2 = projected_tt_vectors_2.tt_cores[0] + curr_du_1 = curr_core_1[:, :, :, :right_size] + curr_du_2 = curr_core_2[:, :, :, :right_size] + res = tf.einsum('paib,qaib->pq', curr_du_1, curr_du_2) + for core_idx in range(1, ndims): + left_size = tt_ranks[core_idx] // 2 + right_size = tt_ranks[core_idx + 1] // 2 + curr_core_1 = projected_tt_vectors_1.tt_cores[core_idx] + curr_core_2 = projected_tt_vectors_2.tt_cores[core_idx] + curr_du_1 = curr_core_1[:, left_size:, :, :right_size] + curr_du_2 = curr_core_2[:, left_size:, :, :right_size] + res += tf.einsum('paib,qaib->pq', curr_du_1, curr_du_2) + + left_size = tt_ranks[-2] // 2 + curr_core_1 = projected_tt_vectors_1.tt_cores[-1] + curr_core_2 = projected_tt_vectors_2.tt_cores[-1] + curr_du_1 = curr_core_1[:, left_size:, :, :] + curr_du_2 = curr_core_2[:, left_size:, :, :] + res += tf.einsum('paib,qaib->pq', curr_du_1, curr_du_2) + return res + + +def add_n_projected(tt_objects, coef=None): + """Adds all input TT-objects that are projections on the same tangent space. + + add_projected((a, b)) is equivalent add(a, b) for a and b that are from the + same tangent space, but doesn't increase the TT-ranks. + + Args: + tt_objects: a list of TT-objects that are projections on the same tangent + space. + coef: a list of numbers or anything else convertable to tf.Tensor. + If provided, computes weighted sum. The size of this array should be + len(tt_objects) x tt_objects[0].batch_size + + Returns: + TT-objects representing the sum of the tt_objects (weighted sum if coef is + provided). The TT-rank of the result equals to the TT-ranks of the arguments. + """ + for tt in tt_objects: + if not hasattr(tt, 'projection_on'): + raise ValueError('Both arguments should be projections on the tangent ' + 'space of some other TT-object. All projection* functions ' + 'leave .projection_on field in the resulting TT-object ' + 'which is not present in the argument you\'ve provided.') + + projection_on = tt_objects[0].projection_on + for tt in tt_objects[1:]: + if tt.projection_on != projection_on: + raise ValueError('All tt_objects should be projections on the tangent ' + 'space of the same TT-object. The provided arguments are ' + 'projections on different TT-objects (%s and %s). Or at ' + 'least the pointers are different.' % (tt.projection_on, + projection_on)) + if coef is not None: + coef = tf.convert_to_tensor(coef) + if coef.get_shape().ndims > 1: + # In batch case we will need to multiply each core by this coefficients + # along the first axis. To do it need to reshape the coefs to match + # the TT-cores number of dimensions. + some_core = tt_objects[0].tt_cores[0] + dim_array = [1] * (some_core.get_shape().ndims + 1) + dim_array[0] = coef.get_shape()[0].value + dim_array[1] = coef.get_shape()[1].value + coef = tf.reshape(coef, dim_array) + + ndims = tt_objects[0].ndims() + tt_ranks = shapes.lazy_tt_ranks(tt_objects[0]) + left_rank_dim = tt_objects[0].left_tt_rank_dim + right_rank_dim = tt_objects[0].right_tt_rank_dim + res_cores = [] + + def slice_tt_core(tt_core, left_idx, right_idx): + num_tt_core_dims = len(tt_core.get_shape()) + idx = [slice(None)] * num_tt_core_dims + idx[left_rank_dim] = left_idx + idx[right_rank_dim] = right_idx + return tt_core[idx] + + right_half_rank = tt_ranks[1] // 2 + left_chunks = [] + for obj_idx, tt in enumerate(tt_objects): + curr_core = slice_tt_core(tt.tt_cores[0], slice(None), + slice(0, right_half_rank)) + if coef is not None: + curr_core *= coef[obj_idx] + left_chunks.append(curr_core) + left_part = tf.add_n(left_chunks) + first_obj_core = tt_objects[0].tt_cores[0] + right_part = slice_tt_core(first_obj_core, slice(None), + slice(right_half_rank, None)) + first_core = tf.concat((left_part, right_part), axis=right_rank_dim) + res_cores.append(first_core) + + for core_idx in range(1, ndims - 1): + first_obj_core = tt_objects[0].tt_cores[core_idx] + left_half_rank = tt_ranks[core_idx] // 2 + right_half_rank = tt_ranks[core_idx + 1] // 2 + + upper_part = slice_tt_core(tt.tt_cores[core_idx], slice(0, left_half_rank), + slice(None)) + lower_right_part = slice_tt_core(first_obj_core, + slice(left_half_rank, None), + slice(right_half_rank, None)) + + lower_left_chunks = [] + for obj_idx, tt in enumerate(tt_objects): + curr_core = slice_tt_core(tt.tt_cores[core_idx], + slice(left_half_rank, None), + slice(0, right_half_rank)) + if coef is not None: + curr_core *= coef[obj_idx] + lower_left_chunks.append(curr_core) + lower_left_part = tf.add_n(lower_left_chunks) + lower_part = tf.concat((lower_left_part, lower_right_part), + axis=right_rank_dim) + curr_core = tf.concat((upper_part, lower_part), axis=left_rank_dim) + res_cores.append(curr_core) + + left_half_rank = tt_ranks[ndims - 1] // 2 + upper_part = slice_tt_core(tt.tt_cores[-1], slice(0, left_half_rank), + slice(None)) + lower_chunks = [] + for obj_idx, tt in enumerate(tt_objects): + curr_core = slice_tt_core(tt.tt_cores[-1], slice(left_half_rank, None), + slice(None)) + if coef is not None: + curr_core *= coef[obj_idx] + lower_chunks.append(curr_core) + lower_part = tf.add_n(lower_chunks) + last_core = tf.concat((upper_part, lower_part), axis=left_rank_dim) + res_cores.append(last_core) + + raw_shape = tt_objects[0].get_raw_shape() + static_tt_ranks = tt_objects[0].get_tt_ranks() + if isinstance(tt_objects[0], TensorTrain): + res = TensorTrain(res_cores, raw_shape, static_tt_ranks) + elif isinstance(tt_objects[0], TensorTrainBatch): + res = TensorTrainBatch(res_cores, raw_shape, static_tt_ranks, + tt_objects[0].batch_size) + # Maintain the projection_on property. + res.projection_on = tt_objects[0].projection_on + return res diff --git a/build/lib/t3f/riemannian_test.py b/build/lib/t3f/riemannian_test.py new file mode 100644 index 00000000..25ddb7d1 --- /dev/null +++ b/build/lib/t3f/riemannian_test.py @@ -0,0 +1,250 @@ +import numpy as np +import tensorflow as tf + +from t3f.tensor_train import TensorTrain +from t3f import ops +from t3f import initializers +from t3f import riemannian +from t3f import shapes +from t3f import batch_ops + + +class RiemannianTest(tf.test.TestCase): + + def testProjectOnItself(self): + # Projection of X into the tangent space of itself is X: P_x(x) = x. + tens = initializers.random_tensor((2, 3, 4)) + proj = riemannian.project_sum(tens, tens) + with self.test_session() as sess: + actual_val, desired_val = sess.run((ops.full(proj), ops.full(tens))) + self.assertAllClose(desired_val, actual_val) + + def testProject(self): + # Compare our projection with the results obtained (and precomputed) from + # tt.riemannian.project which is well tested. + tangent_tens_cores = ([[[-0.42095269, 0.02130842], + [-0.4181081 , 0.42945687], + [ 0.45972439, -0.4525616 ], + [-0.17159869, -0.14505528]]], [[[ 0.23344421], + [ 0.81480049], + [-0.92385135]], + + [[-0.19279465], + [ 0.524976 ], + [-0.40149197]]]) + tangent_tens = TensorTrain(tangent_tens_cores, (4, 3), (1, 2, 1)) + tens_cores = ([[[-1.01761142, 0.36075896, -0.2493624 ], + [-0.99896565, -1.12685474, 1.02832458], + [ 1.08739724, -0.6537435 , 1.99975537], + [ 0.35128005, 0.40395104, -0.16790072]]], [[[ 0.34105142], + [ 0.10371947], + [-1.76464904]], + + [[ 0.32639758], + [-1.27843174], + [-1.41590327]], + + [[ 0.76616274], + [ 0.6577514 ], + [ 2.13703185]]]) + tens = TensorTrain(tens_cores, (4, 3), (1, 3, 1)) + desired_projection = [[-0.67638254, -1.17163914, 0.29850939], + [-1.66479093, -0.99003251, 2.46629195], + [-0.04847773, -0.72908174, 0.20142675], + [ 0.34431125, -0.20935516, -1.15864246]] + proj = riemannian.project_sum(tens, tangent_tens) + with self.test_session() as sess: + self.assertAllClose(desired_projection, ops.full(proj).eval()) + + def testProjectSum(self): + # Test projecting a batch of TT-tensors. + tens = initializers.random_tensor_batch((2, 3, 4), batch_size=3) + tangent_tens = initializers.random_tensor((2, 3, 4), 3) + weighted_sum = tens[0] + tens[1] + tens[2] + direct_proj = riemannian.project_sum(weighted_sum, tangent_tens) + actual_proj = riemannian.project_sum(tens, tangent_tens) + with self.test_session() as sess: + res = sess.run((ops.full(direct_proj), ops.full(actual_proj))) + desired_val, actual_val = res + self.assertAllClose(desired_val, actual_val) + + def testProjectWeightedSum(self): + # Test projecting a batch of TT-tensors with providing coefs. + tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4) + coef = [0.1, -2, 0, 0.4] + tangent_tens = initializers.random_tensor((2, 3, 4), 4) + weighted_sum = coef[0] * tens[0] + coef[1] * tens[1] + coef[2] * tens[2] + weighted_sum += coef[3] * tens[3] + direct_proj = riemannian.project_sum(weighted_sum, tangent_tens) + actual_proj = riemannian.project_sum(tens, tangent_tens, coef) + with self.test_session() as sess: + res = sess.run((ops.full(direct_proj), ops.full(actual_proj))) + desired_val, actual_val = res + self.assertAllClose(desired_val, actual_val) + + def testProjectWeightedSumMultipleOutputs(self): + # Test projecting a batch of TT-tensors with providing weights and outputing + # several TT objects with different weights. + tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4) + np.random.seed(0) + weights = np.random.randn(4, 2).astype(np.float32) + tangent_tens = initializers.random_tensor((2, 3, 4), 4) + weighted_sum_1 = weights[0, 0] * tens[0] + weights[1, 0] * tens[1] +\ + weights[2, 0] * tens[2] + weights[3, 0] * tens[3] + weighted_sum_2 = weights[0, 1] * tens[0] + weights[1, 1] * tens[1] +\ + weights[2, 1] * tens[2] + weights[3, 1] * tens[3] + direct_proj_1 = riemannian.project_sum(weighted_sum_1, tangent_tens) + direct_proj_2 = riemannian.project_sum(weighted_sum_2, tangent_tens) + direct_proj_1 = shapes.expand_batch_dim(direct_proj_1) + direct_proj_2 = shapes.expand_batch_dim(direct_proj_2) + direct_projs = batch_ops.concat_along_batch_dim((direct_proj_1, direct_proj_2)) + actual_proj = riemannian.project_sum(tens, tangent_tens, weights) + with self.test_session() as sess: + res = sess.run((ops.full(direct_projs), ops.full(actual_proj))) + desired_val, actual_val = res + self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) + + def testProjectMatrixOnItself(self): + # Project a TT-matrix on itself. + # Projection of X into the tangent space of itself is X: P_x(x) = x. + tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4))) + proj = riemannian.project_sum(tt_mat, tt_mat) + with self.test_session() as sess: + actual_val, desired_val = sess.run((ops.full(proj), ops.full(tt_mat))) + self.assertAllClose(desired_val, actual_val) + + def testCompareProjectSumAndProject(self): + # Compare results of project_sum and project. + tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4) + tangent_tens = initializers.random_tensor((2, 3, 4), 4) + project_sum = riemannian.project_sum(tens, tangent_tens, tf.eye(4)) + project = riemannian.project(tens, tangent_tens) + with self.test_session() as sess: + res = sess.run((ops.full(project_sum), ops.full(project))) + project_sum_val, project_val = res + self.assertAllClose(project_sum_val, project_val) + + def testProjectMatmul(self): + # Project a TT-matrix times TT-vector on a TT-vector. + tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4))) + tt_vec_what = initializers.random_matrix_batch(((2, 3, 4), None), + batch_size=3) + tt_vec_where = initializers.random_matrix(((2, 3, 4), None)) + proj = riemannian.project_matmul(tt_vec_what, tt_vec_where, tt_mat) + matvec = ops.matmul(tt_mat, tt_vec_what) + proj_desired = riemannian.project(matvec, tt_vec_where) + with self.test_session() as sess: + actual_val, desired_val = sess.run((ops.full(proj), ops.full(proj_desired))) + self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) + + def testPairwiseFlatInnerTensor(self): + # Compare pairwise_flat_inner_projected against naive implementation. + what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3) + what2 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=4) + where = initializers.random_tensor((2, 3, 4), 3) + projected1 = riemannian.project(what1, where) + projected2 = riemannian.project(what2, where) + desired = batch_ops.pairwise_flat_inner(projected1, projected2) + actual = riemannian.pairwise_flat_inner_projected(projected1, projected2) + with self.test_session() as sess: + desired_val, actual_val = sess.run((desired, actual)) + self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) + + with self.assertRaises(ValueError): + # Second argument is not a projection on the tangent space. + riemannian.pairwise_flat_inner_projected(projected1, what2) + where2 = initializers.random_tensor((2, 3, 4), 3) + another_projected2 = riemannian.project(what2, where2) + with self.assertRaises(ValueError): + # The arguments are projections on different tangent spaces. + riemannian.pairwise_flat_inner_projected(projected1, another_projected2) + + def testPairwiseFlatInnerMatrix(self): + # Compare pairwise_flat_inner_projected against naive implementation. + what1 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=3) + what2 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=4) + where = initializers.random_matrix(((2, 3, 4), None), 3) + projected1 = riemannian.project(what1, where) + projected2 = riemannian.project(what2, where) + desired = batch_ops.pairwise_flat_inner(projected1, projected2) + actual = riemannian.pairwise_flat_inner_projected(projected1, projected2) + with self.test_session() as sess: + desired_val, actual_val = sess.run((desired, actual)) + self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) + + with self.assertRaises(ValueError): + # Second argument is not a projection on the tangent space. + riemannian.pairwise_flat_inner_projected(projected1, what2) + where2 = initializers.random_matrix(((2, 3, 4), None), 3) + another_projected2 = riemannian.project(what2, where2) + with self.assertRaises(ValueError): + # The arguments are projections on different tangent spaces. + riemannian.pairwise_flat_inner_projected(projected1, another_projected2) + + def testAddNProjected(self): + # Add several TT-objects from the same tangent space. + what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3) + what2 = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=3) + where = initializers.random_tensor((2, 3, 4), 3) + projected1 = riemannian.project(what1, where) + projected2 = riemannian.project(what2, where) + desired = ops.full(projected1 + projected2) + actual = ops.full(riemannian.add_n_projected((projected1, projected2))) + with self.test_session() as sess: + desired_val, actual_val = sess.run((desired, actual)) + self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) + + with self.assertRaises(ValueError): + # Second argument is not a projection on the tangent space. + riemannian.add_n_projected((projected1, what2)) + where2 = initializers.random_tensor((2, 3, 4), 3) + another_projected2 = riemannian.project(what2, where2) + with self.assertRaises(ValueError): + # The arguments are projections on different tangent spaces. + riemannian.add_n_projected((projected1, another_projected2)) + + def testWeightedAddNProjected(self): + # Add several TT-objects from the same tangent space with coefs. + what1 = initializers.random_tensor((2, 3, 4), 4) + what2 = initializers.random_tensor((2, 3, 4), 1) + where = initializers.random_tensor((2, 3, 4), 3) + projected1 = riemannian.project(what1, where) + projected2 = riemannian.project(what2, where) + desired = ops.full(1.2 * projected1 + -2.0 * projected2) + actual = ops.full(riemannian.add_n_projected((projected1, projected2), + coef=[1.2, -2.0])) + with self.test_session() as sess: + desired_val, actual_val = sess.run((desired, actual)) + self.assertAllClose(desired_val, actual_val) + + with self.assertRaises(ValueError): + # Second argument is not a projection on the tangent space. + riemannian.add_n_projected((projected1, what2), coef=[1.2, -2.0]) + where2 = initializers.random_tensor((2, 3, 4), 3) + another_projected2 = riemannian.project(what2, where2) + with self.assertRaises(ValueError): + # The arguments are projections on different tangent spaces. + riemannian.add_n_projected((projected1, another_projected2), + coef=[1.2, -2.0]) + + def testWeightedAddNProjectedBatch(self): + # Add several TT-batches from the same tangent space with coefs. + what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3) + what2 = initializers.random_tensor_batch((2, 3, 4), 1, batch_size=3) + where = initializers.random_tensor((2, 3, 4), 3) + projected1 = riemannian.project(what1, where) + projected2 = riemannian.project(what2, where) + + desired_0 = ops.full(1.2 * projected1[0] + -2.0 * projected2[0]) + desired_1 = ops.full(1.9 * projected1[1] + 2.0 * projected2[1]) + desired_2 = ops.full(0.0 * projected1[2] + 1.0 * projected2[2]) + desired = tf.stack((desired_0, desired_1, desired_2), axis=0) + actual = ops.full(riemannian.add_n_projected((projected1, projected2), + coef=[[1.2, 1.9, 0.0], + [-2.0, 2.0, 1.0]])) + with self.test_session() as sess: + desired_val, actual_val = sess.run((desired, actual)) + self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/shapes.py b/build/lib/t3f/shapes.py new file mode 100644 index 00000000..bc411092 --- /dev/null +++ b/build/lib/t3f/shapes.py @@ -0,0 +1,281 @@ +import numpy as np +import tensorflow as tf + + +# TODO: test all these functions. +def tt_ranks(tt): + """Returns the TT-ranks of a TensorTrain. + + This operation returns a 1-D integer tensor representing the TT-ranks of + the input. + + Args: + tt: `TensorTrain` or `TensorTrainBatch` object. + + Returns: + A `Tensor` + """ + num_dims = tt.ndims() + ranks = [] + for i in range(num_dims): + ranks.append(tf.shape(tt.tt_cores[i])[tt.left_tt_rank_dim]) + ranks.append(tf.shape(tt.tt_cores[-1])[-1]) + return tf.stack(ranks, axis=0) + + +def shape(tt): + """Returns the shape of a TensorTrain. + + This operation returns a 1-D integer tensor representing the shape of + the input. For TT-matrices the shape would have two values, see raw_shape for + the tensor shape. + If the input is a TensorTrainBatch, the first dimension of the output is the + batch_size. + + Args: + tt: `TensorTrain` or `TensorTrainBatch` object. + + Returns: + A `Tensor` + """ + tt_raw_shape = raw_shape(tt) + if tt.is_tt_matrix(): + res = tf.reduce_prod(tt_raw_shape, axis=1) + else: + res = tt_raw_shape[0] + + # TODO: ugly. + from t3f.tensor_train_batch import TensorTrainBatch + if isinstance(tt, TensorTrainBatch): + res = tf.concat((tf.expand_dims(batch_size(tt), 0), res), axis=0) + + return res + + +def raw_shape(tt): + """Returns the shape of a TensorTrain. + + This operation returns a 2-D integer tensor representing the shape of + the input. + If the input is a TT-tensor, the shape will have 1 x ndims() elements. + If the input is a TT-matrix, the shape will have 2 x ndims() elements + representing the underlying tensor shape of the matrix. + + Args: + tt: `TensorTrain` or `TensorTrainBatch` object. + + Returns: + A 2-D `Tensor` of size 1 x ndims() or 2 x ndims() + """ + num_dims = tt.ndims() + num_tensor_axis = len(tt.get_raw_shape()) + final_raw_shape = [] + # TODO: ugly. + from t3f.tensor_train import TensorTrain + axes_shift = 1 if isinstance(tt, TensorTrain) else 2 + for ax in range(num_tensor_axis): + curr_raw_shape = [] + for core_idx in range(num_dims): + curr_raw_shape.append(tf.shape(tt.tt_cores[core_idx])[ax + axes_shift]) + final_raw_shape.append(tf.stack(curr_raw_shape, axis=0)) + return tf.stack(final_raw_shape, axis=0) + + +def batch_size(tt): + """Return the number of elements in a TensorTrainBatch. + + Return 0-D integer tensor. + + Raises: + ValueError if got `TensorTrain` which doesn't have batch_size as input.""" + if not hasattr(tt, 'batch_size'): + raise ValueError('batch size is not available for a TensorTrain object.') + first_core = tt.tt_cores[0] + # The first dimension of any TT-core in TensorTrainBatch is the batch size. + return tf.shape(first_core)[0] + + +def lazy_tt_ranks(tt): + """Returns static TT-ranks of a TensorTrain if defined, and dynamic otherwise. + + This operation returns a 1-D integer numpy array of TT-ranks if they are + available on the graph compilation stage and 1-D integer tensor of dynamic + TT-ranks otherwise. + + Args: + tt: `TensorTrain` object. + + Returns: + A 1-D numpy array or `tf.Tensor` + """ + static_tt_ranks = tt.get_tt_ranks() + if static_tt_ranks.is_fully_defined(): + return np.array(static_tt_ranks.as_list()) + else: + return tt_ranks(tt) + + +def lazy_shape(tt): + """Returns static shape of a TensorTrain if defined, and dynamic otherwise. + + This operation returns a 1-D integer numpy array representing the shape of the + input if it is available on the graph compilation stage and 1-D integer tensor + of dynamic shape otherwise. + + Args: + tt: `TensorTrain` object. + + Returns: + A 1-D numpy array or `tf.Tensor` + """ + static_shape = tt.get_shape() + if static_shape.is_fully_defined(): + return np.array(static_shape.as_list()) + else: + return shape(tt) + + +def lazy_raw_shape(tt): + """Returns static raw shape of a TensorTrain if defined, and dynamic otherwise. + + This operation returns a 2-D integer numpy array representing the raw shape of + the input if it is available on the graph compilation stage and 2-D integer + tensor of dynamic shape otherwise. + If the input is a TT-tensor, the raw shape will have 1 x ndims() elements. + If the input is a TT-matrix, the raw shape will have 2 x ndims() elements + representing the underlying tensor shape of the matrix. + + Args: + tt: `TensorTrain` object. + + Returns: + A 2-D numpy array or `tf.Tensor` of size 1 x ndims() or 2 x ndims() + """ + # If get_shape is fully defined, it guaranties that all elements of raw shape + # are defined. + if tt.get_shape().is_fully_defined(): + return np.array([s.as_list() for s in tt.get_raw_shape()]) + else: + return raw_shape(tt) + + +def lazy_batch_size(tt): + """Return static batch_size if available and dynamic otherwise. + + Args: + tt: `TensorTrainBatch` object. + + Returns: + A number or a 0-D `tf.Tensor` + + Raises: + ValueError if got `TensorTrain` which doesn't have batch_size as input.""" + if not hasattr(tt, 'batch_size'): + raise ValueError('batch size is not available for a TensorTrain object.') + if tt.batch_size is not None: + return tt.batch_size + else: + return batch_size(tt) + + +def clean_raw_shape(shape): + """Returns a tuple of TensorShapes for any valid shape representation. + + Args: + shape: An np.array, a tf.TensorShape (for tensors), a tuple of + tf.TensorShapes (for TT-matrices or tensors), or None + + Returns: + A tuple of tf.TensorShape, or None if the input is None + """ + if shape is None: + return None + if isinstance(shape, tf.TensorShape) or isinstance(shape[0], tf.TensorShape): + # Assume tf.TensorShape. + if isinstance(shape, tf.TensorShape): + shape = tuple((shape,)) + else: + np_shape = np.array(shape) + # Make sure that the shape is 2-d array both for tensors and TT-matrices. + np_shape = np.squeeze(np_shape) + if len(np_shape.shape) == 1: + # A tensor. + np_shape = [np_shape] + shape = [] + for i in range(len(np_shape)): + shape.append(tf.TensorShape(np_shape[i])) + shape = tuple(shape) + return shape + + +def is_batch_broadcasting_possible(tt_a, tt_b): + """Check that the batch broadcasting possible for the given batch sizes. + + Returns true if the batch sizes are the same or if one of them is 1. + + If the batch size that is supposed to be 1 is not known on compilation stage, + broadcasting is not allowed. + + Args: + tt_a: TensorTrain or TensorTrainBatch + tt_b: TensorTrain or TensorTrainBatch + + Returns: + Bool + """ + try: + if tt_a.batch_size is None and tt_b.batch_size is None: + # If both batch sizes are not available on the compilation stage, + # we cannot say if broadcasting is possible so we will not allow it. + return False + if tt_a.batch_size == tt_b.batch_size: + return True + if tt_a.batch_size == 1 or tt_b.batch_size == 1: + return True + return False + except AttributeError: + # One or both of the arguments are not batch tensor, but single TT tensors. + # In this case broadcasting is always possible. + return True + + +def squeeze_batch_dim(tt): + """Converts batch size 1 TensorTrainBatch into TensorTrain. + + Args: + tt: TensorTrain or TensorTrainBatch. + + Returns: + TensorTrain if the input is a TensorTrainBatch with batch_size == 1 (known + at compilation stage) or a TensorTrain. + TensorTrainBatch otherwise. + """ + try: + if tt.batch_size == 1: + return tt[0] + else: + return tt + except AttributeError: + # tt object does not have attribute batch_size, probably already + # a TensorTrain. + return tt + + +def expand_batch_dim(tt): + """Creates a 1-element TensorTrainBatch from a TensorTrain. + + Args: + tt: TensorTrain or TensorTrainBatch. + + Returns: + TensorTrainBatch + """ + if hasattr(tt, 'batch_size'): + return tt + else: + from t3f.tensor_train_batch import TensorTrainBatch + tt_cores = [] + for core_idx in range(tt.ndims()): + tt_cores.append(tf.expand_dims(tt.tt_cores[core_idx], 0)) + return TensorTrainBatch(tt_cores, tt.get_raw_shape(), tt.get_tt_ranks(), + batch_size=1) diff --git a/build/lib/t3f/tensor_train.py b/build/lib/t3f/tensor_train.py new file mode 100644 index 00000000..b884e92f --- /dev/null +++ b/build/lib/t3f/tensor_train.py @@ -0,0 +1,229 @@ +import tensorflow as tf + +from t3f.tensor_train_base import TensorTrainBase +from t3f import shapes + + +class TensorTrain(TensorTrainBase): + """Represents a Tensor Train object (a TT-tensor or TT-matrix). + + t3f represents a Tensor Train object as a tuple of TT-cores. + ``` + @@__init__ + @@get_raw_shape + @@get_shape + @@tt_cores + @@dtype + @@name + @@graph + @@ndims + @@get_tt_ranks + @@left_tt_rank_dim + @@right_tt_rank_dim + @@is_tt_matrix + @@is_variable + @@eval + """ + + def __init__(self, tt_cores, shape=None, tt_ranks=None, convert_to_tensors=True): + """Creates a `TensorTrain`. + + Args: + tt_cores: A tuple of 3d or 4d tensor-like objects of shape + `[r_k-1, n_k, r_k]`. + Tensor-like can be numpy array, tf.Tensor, of tf.Variable + shape: Shape of the underlying tensor. If None, tries to infer from the + cores (not always possible even if it should be, e.g. if ranks are + unknown, than the whole shape of a core can be unknown). + tt_ranks: a TensorShape of length d+1 (d is the dimensionality of + the underlying tensor). The first and the last ranks are assumed to + equal to 1. If None, tries to infer the ranks from the cores. + convert_to_tensors: bool, if True than convert each element of the + tt_cores tuple into a tf.Tensor (e.g. to initialize from np.array) + + Returns: + A `TensorTrain`. + + Raises: + ValueError if the provided TT-cores are not valid or inconsistent with + the provided shape. + """ + tt_cores = list(tt_cores) + if convert_to_tensors: + # TODO: what does this namescope do? + with tf.name_scope("TensorTrain", tt_cores): + for i in range(len(tt_cores)): + name = "core%d" % i + tt_cores[i] = tf.convert_to_tensor(tt_cores[i], name=name) + + if not _are_tt_cores_valid(tt_cores, shape, tt_ranks): + raise ValueError('The tt_cores provided to TensorTrain constructor are ' + 'not valid, have different dtypes, or are inconsistent ' + 'with the provided shape or TT-ranks.') + + self._tt_cores = tuple(tt_cores) + self._raw_shape = shapes.clean_raw_shape(shape) + if self._raw_shape is None: + self._raw_shape = _infer_raw_shape(self._tt_cores) + self._tt_ranks = None if tt_ranks is None else tf.TensorShape(tt_ranks) + if self._tt_ranks is None: + self._tt_ranks = _infer_tt_ranks(self._tt_cores) + + @property + def tt_cores(self): + """A tuple of TT-cores. + + Returns: + A tuple of 3d or 4d tensors shape + `[r_k-1, n_k, r_k]` + or + `[r_k-1, n_k, m_k, r_k]` + """ + return self._tt_cores + + @property + def left_tt_rank_dim(self): + """The dimension of the left TT-rank in each TT-core.""" + return 0 + + @property + def right_tt_rank_dim(self): + """The dimension of the right TT-rank in each TT-core.""" + if self.is_tt_matrix(): + # The dimensions of each TT-core are + # [left_rank, n, m, right_rank] + return 3 + else: + # The dimensions of each TT-core are + # [left_rank, n, right_rank] + return 2 + + def __str__(self): + """A string describing the TensorTrain object, its TT-rank, and shape.""" + shape = self.get_shape() + tt_ranks = self.get_tt_ranks() + variable_str = ' variable' if self.is_variable() else '' + if self.is_tt_matrix(): + raw_shape = self.get_raw_shape() + return "A TT-Matrix%s of size %d x %d, underlying tensor " \ + "shape: %s x %s, TT-ranks: %s" % (variable_str, shape[0], shape[1], + raw_shape[0], raw_shape[1], + tt_ranks) + else: + return "A Tensor Train%s of shape %s, TT-ranks: %s" % (variable_str, + shape, tt_ranks) + + def __getitem__(self, slice_spec): + """Basic indexing, returns a `TensorTrain` containing the specified region. + + Examples: + >>> a = t3f.random_tensor((2, 3, 4)) + >>> a[1, :, :] + is a 2D TensorTrain 3 x 4. + >>> a[1:2, :, :] + is a 3D TensorTrain 1 x 3 x 4 + """ + if len(slice_spec) != self.ndims(): + raise ValueError('Expected %d indices, got %d' % (self.ndims(), + len(slice_spec))) + new_tt_cores = [] + remainder = None + for i in range(self.ndims()): + curr_core = self.tt_cores[i] + if self.is_tt_matrix(): + raise NotImplementedError + else: + sliced_core = curr_core[:, slice_spec[i], :] + if len(curr_core.get_shape()) != len(sliced_core.get_shape()): + # This index is specified exactly and we want to collapse this axis. + if remainder is None: + remainder = sliced_core + else: + remainder = tf.matmul(remainder, sliced_core) + else: + if remainder is not None: + # Add reminder from the previous collapsed cores to the current + # core. + sliced_core = tf.einsum('ab,bid->aid', remainder, sliced_core) + remainder = None + new_tt_cores.append(sliced_core) + + if remainder is not None: + # The reminder obtained from collapsing the last cores. + new_tt_cores[-1] = tf.einsum('aib,bd->aid', new_tt_cores[-1], remainder) + remainder = None + # TODO: infer the output ranks and shape. + return TensorTrain(new_tt_cores) + + +def _are_tt_cores_valid(tt_cores, shape, tt_ranks): + """Check if dimensions of the TT-cores are consistent and the dtypes coincide. + + Args: + tt_cores: a tuple of `Tensor` objects + shape: An np.array, a tf.TensorShape (for tensors), a tuple of + tf.TensorShapes (for TT-matrices or tensors), or None + tt_ranks: An np.array or a tf.TensorShape of length len(tt_cores)+1. + + Returns: + boolean, True if the dimensions and dtypes are consistent. + """ + shape = shapes.clean_raw_shape(shape) + num_dims = len(tt_cores) + + for core_idx in range(1, num_dims): + if tt_cores[core_idx].dtype != tt_cores[0].dtype: + return False + try: + for core_idx in range(num_dims): + curr_core_shape = tt_cores[core_idx].get_shape() + if len(curr_core_shape) != len(tt_cores[0].get_shape()): + # Shapes are inconsistent. + return False + if shape is not None: + for i in range(len(shape)): + if curr_core_shape[i + 1] != shape[i][core_idx]: + # The TT-cores are not aligned with the given shape. + return False + if core_idx >= 1: + prev_core_shape = tt_cores[core_idx - 1].get_shape() + if curr_core_shape[0] != prev_core_shape[-1]: + # TT-ranks are inconsistent. + return False + if tt_ranks is not None: + if curr_core_shape[0] != tt_ranks[core_idx]: + # The TT-ranks are not aligned with the TT-cores shape. + return False + if curr_core_shape[-1] != tt_ranks[core_idx + 1]: + # The TT-ranks are not aligned with the TT-cores shape. + return False + if tt_cores[0].get_shape()[0] != 1 or tt_cores[-1].get_shape()[-1] != 1: + # The first or the last rank is not 1. + return False + except ValueError: + # The shape of the TT-cores is undetermined, can not validate it. + pass + return True + + +def _infer_raw_shape(tt_cores): + """Tries to infer the (static) raw shape from the TT-cores.""" + num_dims = len(tt_cores) + num_tensor_shapes = len(tt_cores[0].get_shape().as_list()) - 2 + raw_shape = [[] for _ in range(num_tensor_shapes)] + for dim in range(num_dims): + curr_core_shape = tt_cores[dim].get_shape() + for i in range(num_tensor_shapes): + raw_shape[i].append(curr_core_shape[i + 1]) + for i in range(num_tensor_shapes): + raw_shape[i] = tf.TensorShape(raw_shape[i]) + return tuple(raw_shape) + + +def _infer_tt_ranks(tt_cores): + """Tries to infer the (static) raw shape from the TT-cores.""" + tt_ranks = [] + for i in range(len(tt_cores)): + tt_ranks.append(tt_cores[i].get_shape()[0]) + tt_ranks.append(tt_cores[-1].get_shape()[-1]) + return tf.TensorShape(tt_ranks) diff --git a/build/lib/t3f/tensor_train_base.py b/build/lib/t3f/tensor_train_base.py new file mode 100644 index 00000000..7bad0a07 --- /dev/null +++ b/build/lib/t3f/tensor_train_base.py @@ -0,0 +1,189 @@ +import numpy as np +import tensorflow as tf + + +# TODO: check the methods of _TensorLike +class TensorTrainBase(object): + """An abstract class that represents a collection of Tensor Train cores. + ``` + @@__init__ + @@get_raw_shape + @@get_shape + @@tt_cores + @@dtype + @@name + @@graph + @@ndims + @@get_tt_ranks + @@is_tt_matrix + @@is_variable + @@eval + """ + + def __init__(self, tt_cores): + """Creates a `TensorTrainBase`.""" + pass + + def get_raw_shape(self): + """Get tuple of `TensorShapes` representing the shapes of the underlying TT-tensor. + + Tuple contains one `TensorShape` for TT-tensor and 2 `TensorShapes` for + TT-matrix + + Returns: + A tuple of `TensorShape` objects. + """ + return self._raw_shape + + def get_shape(self): + """Get the `TensorShape` representing the shape of the dense tensor. + + Returns: + A `TensorShape` object. + """ + raw_shape = self.get_raw_shape() + if self.is_tt_matrix(): + # TODO: as list is not available if shape is partly known. + m = np.prod(raw_shape[0].as_list()) + n = np.prod(raw_shape[1].as_list()) + return tf.TensorShape((m, n)) + else: + return raw_shape[0] + + @property + def tt_cores(self): + """A tuple of TT-cores.""" + return self._tt_cores + + @property + def dtype(self): + """The `DType` of elements in this tensor.""" + # TODO: where is this created? + return self.tt_cores[0].dtype + + @property + def name(self): + """The name of the TensorTrain. + + Returns: + String, the scope in which the TT-cores are defined. + """ + core_name = self.tt_cores[0].name + idx = core_name.rfind('/') + return core_name[:idx] + + @property + def graph(self): + """The `Graph` that contains the tt_cores tensors.""" + # TODO: check in init that the other cores are from the same graph. + return self.tt_cores[0].graph + + def __str__(self): + """A string describing the TensorTrain object, its TT-rank and shape.""" + return NotImplementedError + + def ndims(self): + """Get the number of dimensions of the underlying TT-tensor. + + Returns: + A number. + """ + return len(self.tt_cores) + + def get_tt_ranks(self): + """Get the TT-ranks in an array of size `num_dims`+1. + + The first and the last TT-rank are guarantied to be 1. + + Returns: + TensorShape of size `num_dims`+1. + """ + return self._tt_ranks + + def is_tt_matrix(self): + """Returns True if the TensorTrain object represents a TT-matrix.""" + return len(self.get_raw_shape()) == 2 + + def is_variable(self): + """True if the TensorTrain object is a variable (e.g. is trainable).""" + return isinstance(self.tt_cores[0], tf.Variable) + + @property + def op(self): + """The `Operation` that evaluates all the cores.""" + return tf.group(*[c.op for c in self.tt_cores]) + + def eval(self, feed_dict=None, session=None): + """Evaluates this sparse tensor in a `Session`. + + Calling this method will execute all preceding operations that + produce the inputs needed for the operation that produces this + tensor. + *N.B.* Before invoking `SparseTensor.eval()`, its graph must have been + launched in a session, and either a default session must be + available, or `session` must be specified explicitly. + + Args: + feed_dict: A dictionary that maps `Tensor` objects to feed values. + See [`Session.run()`](../../api_docs/python/client.md#Session.run) for a + description of the valid feed values. + session: (Optional.) The `Session` to be used to evaluate this sparse + tensor. If none, the default session will be used. + """ + # TODO: implement feed_dict + if session is None: + session = tf.get_default_session() + session.run(self.tt_cores) + + # TODO: do we need this? + # @staticmethod + # def _override_operator(operator, func): + # _override_helper(SparseTensor, operator, func) + + def __add__(self, other): + """Returns a TensorTrain corresponding to element-wise sum tt_a + tt_b. + + Supports broadcasting (e.g. you can add TensorTrainBatch and TensorTrain). + Just calls t3f.add, see its documentation for details. + """ + # TODO: ugly. + # We can't import ops in the beginning since it creates cyclic dependencies. + from t3f import ops + return ops.add(self, other) + + def __sub__(self, other): + """Returns a TensorTrain corresponding to element-wise difference tt_a - tt_b. + + Supports broadcasting (e.g. you can subtract TensorTrainBatch and + TensorTrain). + Just calls t3f.add(self, (-1) * other), see its documentation for details. + """ + # TODO: ugly. + # We can't import ops in the beginning since it creates cyclic dependencies. + from t3f import ops + return ops.add(self, ops.multiply(other, -1.)) + + def __neg__(self): + """Returns a TensorTrain corresponding to element-wise negative -tt_a. + + Just calls t3f.multiply(self, -1.), see its documentation for details. + """ + # TODO: ugly. + # We can't import ops in the beginning since it creates cyclic dependencies. + from t3f import ops + return ops.multiply(self, -1.) + + def __mul__(self, other): + """Returns a TensorTrain corresponding to element-wise product tt_a * tt_b. + + Supports broadcasting (e.g. you can multiply TensorTrainBatch and + TensorTrain). + Just calls t3f.multiply, see its documentation for details. + """ + # TODO: ugly. + # We can't import ops in the beginning since it creates cyclic dependencies. + from t3f import ops + return ops.multiply(self, other) + + # To support 'TT * 4' as well as '4 * TT'. + __rmul__ = __mul__ diff --git a/build/lib/t3f/tensor_train_batch.py b/build/lib/t3f/tensor_train_batch.py new file mode 100644 index 00000000..5ff0410b --- /dev/null +++ b/build/lib/t3f/tensor_train_batch.py @@ -0,0 +1,367 @@ +import numpy as np +import tensorflow as tf + +from t3f.tensor_train_base import TensorTrainBase +from t3f.tensor_train import TensorTrain +from t3f import shapes + + +class TensorTrainBatch(TensorTrainBase): + """Represents a batch of Tensor Train objects (TT-tensors or TT-matrices). + + t3f represents a Tensor Train object as a tuple of TT-cores. + ``` + @@__init__ + @@get_raw_shape + @@get_shape + @@tt_cores + @@dtype + @@name + @@graph + @@ndims + @@get_tt_ranks + @@left_tt_rank_dim + @@right_tt_rank_dim + @@is_tt_matrix + @@is_variable + @@eval + """ + + def __init__(self, tt_cores, shape=None, tt_ranks=None, batch_size=None, + convert_to_tensors=True): + """Creates a `TensorTrainBatch`. + + Args: + tt_cores: A tuple of 4d or 5d tensor-like objects of shape + `[batch_size, r_k-1, n_k, r_k]` or + `[batch_size, r_k-1, n_k, m_k, r_k]` + Tensor-like can be numpy array, tf.Tensor, of tf.Variable + batch_size: number of elements in the batch. If None, tries to infer from + the TT-cores (not always possible even if it should be, e.g. if ranks + are unknown, than the whole shape of a core can be unknown). + shape: Shape of the underlying tensor. If None, tries to infer from the + TT-cores. + tt_ranks: a TensorShape of length d+1 (d is the dimensionality of + the underlying tensor). The first and the last ranks are assumed to + equal to 1. If None, tries to infer the ranks from the cores. + convert_to_tensors: bool, if True than convert each element of the + tt_cores tuple into a tf.Tensor (e.g. to initialize from np.array) + + Returns: + A `TensorTrainBatch`. + + Raises: + ValueError if the provided TT-cores are not valid or inconsistent with + the provided shape. + """ + tt_cores = list(tt_cores) + if convert_to_tensors: + # TODO: what does this namescope do? + with tf.name_scope("TensorTrainBatch", tt_cores): + for i in range(len(tt_cores)): + name = "core%d" % i + tt_cores[i] = tf.convert_to_tensor(tt_cores[i], name=name) + + if not _are_batch_tt_cores_valid(tt_cores, shape, tt_ranks, batch_size): + raise ValueError('The tt_cores provided to TensorTrainBatch constructor ' + 'are not valid, have different dtypes, or are ' + 'inconsistent with the provided batch_size, shape, or ' + 'TT-ranks.') + + self._tt_cores = tuple(tt_cores) + if batch_size is None: + self._batch_size = tt_cores[0].get_shape()[0].value + else: + self._batch_size = batch_size + self._raw_shape = shapes.clean_raw_shape(shape) + if self._raw_shape is None: + self._raw_shape = _infer_batch_raw_shape(self._tt_cores) + self._tt_ranks = None if tt_ranks is None else tf.TensorShape(tt_ranks) + if self._tt_ranks is None: + self._tt_ranks = _infer_batch_tt_ranks(self._tt_cores) + + def get_shape(self): + """Get the `TensorShape` representing the shape of the dense tensor. + + The first dimension is the batch_size. + + Returns: + A `TensorShape` object. + """ + shape = TensorTrainBase.get_shape(self) + return tf.TensorShape(np.hstack((self.batch_size, shape))) + + @property + def tt_cores(self): + """A tuple of TT-cores. + + Returns: + A tuple of 4d or 5d tensors shape + `[batch_size, r_k-1, n_k, r_k]` + or + `[batch_size, r_k-1, n_k, m_k, r_k]` + """ + return self._tt_cores + + @property + def batch_size(self): + """The number of elements or None if not known.""" + return self._batch_size + + @property + def left_tt_rank_dim(self): + """The dimension of the left TT-rank in each TT-core.""" + return 1 + + @property + def right_tt_rank_dim(self): + """The dimension of the right TT-rank in each TT-core.""" + if self.is_tt_matrix(): + # The dimensions of each TT-core are + # [batch_idx, left_rank, n, m, right_rank] + return 4 + else: + # The dimensions of each TT-core are + # [batch_idx, left_rank, n, right_rank] + return 3 + + def __str__(self): + """A string describing the TensorTrainBatch, its TT-rank and shape.""" + shape = self.get_shape() + tt_ranks = self.get_tt_ranks() + + if self.batch_size is None: + batch_size_str = '(?)' + else: + batch_size_str = str(self.batch_size) + + if self.is_tt_matrix(): + raw_shape = self.get_raw_shape() + type_str = 'TT-matrix variables' if self.is_variable() else 'TT-matrices' + + return "A %s element batch of %s of size %d x %d, underlying tensor " \ + "shape: %s x %s, TT-ranks: %s" % (batch_size_str, type_str, + shape[1], shape[2], + raw_shape[0], raw_shape[1], + tt_ranks) + else: + if self.is_variable(): + type_str = 'Tensor Train variables' + else: + type_str = 'Tensor Trains' + return "A %s element batch of %s of shape %s, TT-ranks: %s" % \ + (batch_size_str, type_str, shape[1:], tt_ranks) + + @staticmethod + def _do_collapse_dim(slice_spec): + # Returns true if slice_spec is specified exactly and we want to collapse + # the corresponding axis, i.e. return an object with less dims. To be used + # in indexing functions. + # If its a actual slice, nothing to collapse. Otherwise (a number or + # a tf.Tensor) want to collapse. + return not isinstance(slice_spec, slice) + + def _batch_dim_getitem(self, element_spec): + """__getitem__ when provided only one (batch) index. + + Examples: + a[1] + a[1:3] + """ + + # This object index is specified exactly and we want to collapse the + # batch_size axis, i.e. return a TensorTrain instead of a TensorTrainBatch. + do_collapse_batch_dim = self._do_collapse_dim(element_spec) + + new_tt_cores = [] + for core_idx in range(self.ndims()): + curr_core = self.tt_cores[core_idx] + if self.is_tt_matrix(): + new_tt_cores.append(curr_core[element_spec, :, :, :, :]) + else: + new_tt_cores.append(curr_core[element_spec, :, :, :]) + if do_collapse_batch_dim: + # This index is specified exactly and we want to collapse the batch_size + # axis, i.e. return a TensorTrain instead of a TensorTrainBatch. + return TensorTrain(new_tt_cores, self.get_raw_shape(), + self.get_tt_ranks()) + else: + batch_size = new_tt_cores[0].get_shape()[0].value + return TensorTrainBatch(new_tt_cores, self.get_raw_shape(), + self.get_tt_ranks(), batch_size) + + def _full_getitem(self, slice_spec): + """__getitem__ when provided full index of length ndims + 1. + + Examples: + a = t3f.random_tensor_batch((2, 3, 4), batch_size=5) + a[:3, 1:2, 4, :] + """ + if len(slice_spec) != self.ndims() + 1: + raise ValueError('Expected %d indices, got %d' % (self.ndims() + 1, + len(slice_spec))) + # This object index is specified exactly and we want to collapse the + # batch_size axis, i.e. return a TensorTrain instead of a TensorTrainBatch. + do_collapse_batch_dim = self._do_collapse_dim(slice_spec[0]) + remainder = None + new_tt_cores = [] + for core_idx in range(self.ndims()): + curr_core = self.tt_cores[core_idx] + if self.is_tt_matrix(): + raise NotImplementedError + else: + sliced_core = curr_core[slice_spec[0], :, slice_spec[core_idx + 1], :] + do_collapse_curr_dim = self._do_collapse_dim(slice_spec[core_idx + 1]) + if do_collapse_curr_dim: + # This index is specified exactly and we want to collapse this axis. + if remainder is None: + remainder = sliced_core + else: + if do_collapse_batch_dim: + remainder = tf.einsum('ab,bd->ad', remainder, sliced_core) + else: + remainder = tf.einsum('oab,obd->oad', remainder, sliced_core) + else: + if remainder is not None: + # Add reminder from the previous collapsed cores to the current + # core. + if do_collapse_batch_dim: + sliced_core = tf.einsum('ab,bid->aid', remainder, sliced_core) + else: + sliced_core = tf.einsum('oab,obid->oaid', remainder, + sliced_core) + remainder = None + new_tt_cores.append(sliced_core) + + if remainder is not None: + # The reminder obtained from collapsing the last cores. + if do_collapse_batch_dim: + new_tt_cores[-1] = tf.einsum('aib,bd->aid', new_tt_cores[-1], + remainder) + + else: + new_tt_cores[-1] = tf.einsum('oaib,obd->oaid', new_tt_cores[-1], + remainder) + remainder = None + # TODO: infer the output ranks and shape. + if do_collapse_batch_dim: + return TensorTrain(new_tt_cores) + else: + return TensorTrainBatch(new_tt_cores) + + def __getitem__(self, slice_spec): + """Basic indexing, returns a `TensorTrainBatch` with the specified region. + + Examples: + >>> a = t3f.random_tensor_batch((2, 3, 4), batch_size=5) + >>> a[1:3, :, :, :] + is a 3D TensorTrainBatch 2 x 3 x 4 with batch_size = 2. + >>> a[1:3] + the same as above, a 3D TensorTrainBatch 2 x 3 x 4 with batch_size = 2. + >>> a[1, :, :, :] + is a 3D TensorTrain 2 x 3 x 4. + >>> a[1] + the same as above, a 3D TensorTrain 2 x 3 x 4. + >>> a[1:3, :, 1, :] + is a 2D TensorTrainBatch 2 x 4 with batch_size = 2. + >>> a[1, :, 1, :] + is a 2D TensorTrain 2 x 4. + + Returns: + `TensorTrainBatch` or `TensorTrain` depending on whether the first + (batch) dim was specified as a range or as a number. + """ + try: + slice_only_batch_dim = len(slice_spec) == 1 + except TypeError: + # The argument is not iterable, so it's a single slice, or a number, or a + # tf.Tensor with a number. + slice_only_batch_dim = True + + if slice_only_batch_dim: + # Indexing only for the batch_size axis, e.g. a[1:3]. + return self._batch_dim_getitem(slice_spec) + elif len(slice_spec) == self.ndims() + 1: + return self._full_getitem(slice_spec) + else: + raise ValueError('TensorTrainBatch.__getitem__: wrong number of ' + 'dimensions, expected 1 or %d, got %d' % + (self.ndims() + 1, len(slice_spec))) + + +def _are_batch_tt_cores_valid(tt_cores, shape, tt_ranks, batch_size): + """Check if dimensions of the TT-cores are consistent and the dtypes coincide. + + Args: + tt_cores: a tuple of `Tensor` objects + shape: An np.array, a tf.TensorShape (for tensors), a tuple of + tf.TensorShapes (for TT-matrices or tensors), or None + tt_ranks: An np.array or a tf.TensorShape of length len(tt_cores)+1. + batch_size: a number or None + + Returns: + boolean, True if the dimensions and dtypes are consistent. + """ + shape = shapes.clean_raw_shape(shape) + num_dims = len(tt_cores) + + for core_idx in range(1, num_dims): + if tt_cores[core_idx].dtype != tt_cores[0].dtype: + return False + try: + for core_idx in range(num_dims): + curr_core_shape = tt_cores[core_idx].get_shape() + if len(curr_core_shape) != len(tt_cores[0].get_shape()): + # Shapes are inconsistent. + return False + if batch_size is not None and curr_core_shape[0].value is not None: + if curr_core_shape[0].value != batch_size: + # The TT-cores are not aligned with the given batch_size. + return False + if shape is not None: + for i in range(len(shape)): + if curr_core_shape[i + 2] != shape[i][core_idx]: + # The TT-cores are not aligned with the given shape. + return False + if core_idx >= 1: + prev_core_shape = tt_cores[core_idx - 1].get_shape() + if curr_core_shape[1] != prev_core_shape[-1]: + # TT-ranks are inconsistent. + return False + if tt_ranks is not None: + if curr_core_shape[1] != tt_ranks[core_idx]: + # The TT-ranks are not aligned with the TT-cores shape. + return False + if curr_core_shape[-1] != tt_ranks[core_idx + 1]: + # The TT-ranks are not aligned with the TT-cores shape. + return False + if tt_cores[0].get_shape()[1] != 1 or tt_cores[-1].get_shape()[-1] != 1: + # The first or the last rank is not 1. + return False + except ValueError: + # The shape of the TT-cores is undetermined, can not validate it. + pass + return True + + +def _infer_batch_raw_shape(tt_cores): + """Tries to infer the (static) raw shape from the TT-cores.""" + num_dims = len(tt_cores) + num_tensor_shapes = len(tt_cores[0].get_shape().as_list()) - 3 + raw_shape = [[] for _ in range(num_tensor_shapes)] + for dim in range(num_dims): + curr_core_shape = tt_cores[dim].get_shape() + for i in range(num_tensor_shapes): + raw_shape[i].append(curr_core_shape[i + 2]) + for i in range(num_tensor_shapes): + raw_shape[i] = tf.TensorShape(raw_shape[i]) + return tuple(raw_shape) + + +def _infer_batch_tt_ranks(tt_cores): + """Tries to infer the (static) raw shape from the TT-cores.""" + tt_ranks = [] + for i in range(len(tt_cores)): + tt_ranks.append(tt_cores[i].get_shape()[1]) + tt_ranks.append(tt_cores[-1].get_shape()[-1]) + return tf.TensorShape(tt_ranks) diff --git a/build/lib/t3f/tensor_train_batch_test.py b/build/lib/t3f/tensor_train_batch_test.py new file mode 100644 index 00000000..b609f1a7 --- /dev/null +++ b/build/lib/t3f/tensor_train_batch_test.py @@ -0,0 +1,77 @@ +import tensorflow as tf + +from t3f import initializers +from t3f import ops + + +class TensorTrainBatchTest(tf.test.TestCase): + + def testTensorIndexing(self): + tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3) + with self.test_session() as sess: + desired = ops.full(tens)[:, :, :, :] + actual = ops.full(tens[:, :, :, :]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[1:3, :, :, :] + actual = ops.full(tens[1:3]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[1, :, :, :] + actual = ops.full(tens[1]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[2, 1, :, :] + actual = ops.full(tens[2, 1, :, :]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[2, 1:2, 1, :] + actual = ops.full(tens[2, 1:2, 1, :]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[1:2, 0:3, :, 3] + actual = ops.full(tens[1:2, 0:3, :, 3]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[:, 1, :, 3] + actual = ops.full(tens[:, 1, :, 3]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + + # Wrong number of dims. + with self.assertRaises(ValueError): + tens[1, :, 3] + with self.assertRaises(ValueError): + tens[1, :, 3, 1:2, 1:3] + with self.assertRaises(ValueError): + tens[1, 1] + + def testPlaceholderTensorIndexing(self): + tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3) + with self.test_session() as sess: + start = tf.placeholder(tf.int32) + end = tf.placeholder(tf.int32) + + desired = ops.full(tens)[0:-1] + actual = ops.full(tens[start:end]) + desired, actual = sess.run([desired, actual], {start: 0, end: -1}) + self.assertAllClose(desired, actual) + + desired = ops.full(tens)[0:1] + actual = ops.full(tens[start:end]) + desired, actual = sess.run([desired, actual], {start: 0, end: 1}) + self.assertAllClose(desired, actual) + + desired = ops.full(tens)[1] + actual = ops.full(tens[start]) + desired, actual = sess.run([desired, actual], {start: 1}) + self.assertAllClose(desired, actual) + + desired = ops.full(tens)[1, 1:3, 1, :3] + actual = ops.full(tens[start, start:end, start, :end]) + desired, actual = sess.run([desired, actual], {start: 1, end: 3}) + self.assertAllClose(desired, actual) + + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/tensor_train_test.py b/build/lib/t3f/tensor_train_test.py new file mode 100644 index 00000000..22983424 --- /dev/null +++ b/build/lib/t3f/tensor_train_test.py @@ -0,0 +1,138 @@ +import tensorflow as tf + +from t3f import tensor_train +from t3f import initializers +from t3f import ops + + +class TensorTrainTest(tf.test.TestCase): + + def testValidateTTCores2d(self): + schedule = (((1, 1, 1, 1), (1, 1, 1), True), + ((1, 1, 1, 1), None, True), + ((1, 1, 1, 1), (1, 2, 1), False), + ((1, 1, 1, 1), (2, 1, 1), False), + ((1, 2, 2, 1), (1, 2, 1), True), + ((1, 2, 2, 1), (1, 1, 1), False), + ((1, 2, 2, 1), (1, 1, 2), False), + ((1, 2, 2, 1), None, True), + ((2, 1, 1, 1), None, False), + ((2, 1, 1, 1), (1, 1, 1), False), + ((1, 1, 1, 2), None, False), + ((1, 1, 1, 2), (1, 1, 1), False), + ((1, 1, 2, 1), None, False), + ((1, 1, 2, 1), (1, 2, 1), False), + ((1, 2, 1, 1), None, False), + ((1, 2, 1, 1), (1, 2, 1), False)) + + for tt_ranks, claimed_tt_ranks, desired in schedule: + a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1])) + b = tf.random_normal((tt_ranks[2], 9, tt_ranks[3])) + with self.test_session(): + actual = tensor_train._are_tt_cores_valid((a, b), (10, 9), + claimed_tt_ranks) + self.assertEqual(desired, actual) + # Wrong shape. + actual = tensor_train._are_tt_cores_valid((a, b), (9, 9), + claimed_tt_ranks) + self.assertEqual(False, actual) + if not desired: + with self.assertRaises(ValueError): + tensor_train.TensorTrain((a, b), (10, 9), claimed_tt_ranks) + + # Make dtypes inconsistent. + b_new = tf.cast(b, tf.float64) + actual = tensor_train._are_tt_cores_valid((a, b_new), (10, 9), + claimed_tt_ranks) + self.assertEqual(False, actual) + with self.assertRaises(ValueError): + tensor_train.TensorTrain((a, b_new), (10, 9), claimed_tt_ranks) + + def testValidateTTCores3d(self): + schedule = (((1, 1, 1, 1, 1, 1), (1, 1, 1, 1), True), + ((1, 1, 1, 1, 1, 1), None, True), + ((1, 1, 1, 1, 1, 1), (1, 1, 1, 2), False), + ((1, 1, 1, 1, 1, 1), (1, 1, 2, 1), False), + ((1, 2, 2, 2, 2, 1), (1, 2, 2, 1), True), + ((1, 2, 2, 2, 2, 1), None, True), + ((1, 2, 2, 2, 2, 1), (1, 2, 1, 1), False), + ((2, 1, 1, 1, 1, 1), None, False), + ((2, 1, 1, 1, 1, 1), (2, 1, 1, 1), False), + ((1, 1, 1, 1, 1, 2), None, False), + ((1, 1, 1, 1, 1, 2), (1, 1, 1, 2), False), + ((1, 1, 2, 1, 1, 1), None, False), + ((1, 1, 2, 1, 1, 1), (1, 2, 1, 1), False), + ((1, 2, 1, 1, 1, 1), None, False), + ((1, 2, 1, 1, 1, 1), (1, 2, 1, 1), False), + ((1, 2, 2, 1, 1, 1), (1, 2, 1, 1), True), + ((1, 2, 2, 1, 1, 1), (1, 2, 2, 1), False), + ((1, 2, 2, 1, 1, 1), None, True), + ((1, 2, 2, 3, 3, 1), (1, 2, 3, 1), True), + ((1, 2, 2, 3, 3, 1), None, True)) + + for tt_ranks, claimed_tt_ranks, desired in schedule: + a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1])) + b = tf.random_normal((tt_ranks[2], 1, tt_ranks[3])) + c = tf.random_normal((tt_ranks[4], 2, tt_ranks[5])) + with self.test_session(): + actual = tensor_train._are_tt_cores_valid((a, b, c), (10, 1, 2), + claimed_tt_ranks) + self.assertEqual(desired, actual) + # Wrong shape. + actual = tensor_train._are_tt_cores_valid((a, b, c), (10, 1, 1), + claimed_tt_ranks) + self.assertEqual(False, actual) + if not desired: + with self.assertRaises(ValueError): + tensor_train.TensorTrain((a, b, c), (10, 1, 2), claimed_tt_ranks) + + # Make dtypes inconsistent. + b_new = tf.cast(b, tf.float64) + actual = tensor_train._are_tt_cores_valid((a, b_new, c), (10, 1, 2), + claimed_tt_ranks) + self.assertEqual(False, actual) + with self.assertRaises(ValueError): + tensor_train.TensorTrain((a, b_new, c), (10, 1, 2), claimed_tt_ranks) + + def testTensorIndexing(self): + tens = initializers.random_tensor((3, 3, 4)) + with self.test_session() as sess: + desired = ops.full(tens)[:, :, :] + actual = ops.full(tens[:, :, :]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[1, :, :] + actual = ops.full(tens[1, :, :]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[1:2, 1, :] + actual = ops.full(tens[1:2, 1, :]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[0:3, :, 3] + actual = ops.full(tens[0:3, :, 3]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + desired = ops.full(tens)[1, :, 3] + actual = ops.full(tens[1, :, 3]) + desired, actual = sess.run([desired, actual]) + self.assertAllClose(desired, actual) + + # Wrong number of dims. + with self.assertRaises(ValueError): + tens[1, :, 3, :] + with self.assertRaises(ValueError): + tens[1, 1] + + def testPlaceholderTensorIndexing(self): + tens = initializers.random_tensor((3, 3, 4)) + with self.test_session() as sess: + start = tf.placeholder(tf.int32) + end = tf.placeholder(tf.int32) + desired = ops.full(tens)[1:3, 1, :3] + actual = ops.full(tens[start:end, start, :end]) + desired, actual = sess.run([desired, actual], {start: 1, end: 3}) + self.assertAllClose(desired, actual) + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/utils.py b/build/lib/t3f/utils.py new file mode 100644 index 00000000..7d26ab0d --- /dev/null +++ b/build/lib/t3f/utils.py @@ -0,0 +1,39 @@ +import numpy as np +import tensorflow as tf + + +# TODO: substitute with native implementation when it's ready. +# https://github.com/tensorflow/tensorflow/issues/2075 +def unravel_index(indices, shape): + with tf.name_scope('unravel_index'): + indices = tf.expand_dims(indices, 0) + shape = tf.expand_dims(shape, 1) + strides_shifted = tf.cumprod(shape, exclusive=True, reverse=True) + res = (indices // strides_shifted) % shape + return tf.transpose(res, (1, 0)) + + +# TODO: get rid of this when TF fixes the NaN bugs in tf.svd: +# https://github.com/tensorflow/tensorflow/issues/8905 +def replace_tf_svd_with_np_svd(): + """Replaces tf.svd with np.svd. Slow, but a workaround for tf.svd bugs.""" + if hasattr(tf, 'original_svd'): + # This function has been already called and tf.svd is already replaced. + return + tf.original_svd = tf.svd + + def my_svd(tensor, full_matrices=False, compute_uv=True): + dtype = tensor.dtype + u, s, v = tf.py_func(np.linalg.svd, [tensor, full_matrices, compute_uv], + [dtype, dtype, dtype]) + s_, u_, v_ = tf.original_svd(tensor, full_matrices, compute_uv) + s = tf.reshape(s, s_.get_shape()) + u = tf.reshape(u, u_.get_shape()) + v = tf.reshape(v, v_.get_shape()) + # Converting numpy order of v dims to TF order. + order = range(tensor.get_shape().ndims) + order[-2], order[-1] = order[-1], order[-2] + v = tf.transpose(v, order) + return s, u, v + + tf.svd = my_svd diff --git a/build/lib/t3f/utils_test.py b/build/lib/t3f/utils_test.py new file mode 100644 index 00000000..170ab420 --- /dev/null +++ b/build/lib/t3f/utils_test.py @@ -0,0 +1,26 @@ +import numpy as np +import tensorflow as tf + +from t3f import utils + + +class UtilsTest(tf.test.TestCase): + + def testUnravelIndex(self): + with self.test_session(): + # 2D. + shape = (7, 6) + linear_idx = [22, 41, 37] + desired = [[3, 4], [6, 5], [6, 1]] + actual = utils.unravel_index(linear_idx, shape) + self.assertAllEqual(desired, actual.eval()) + # 3D. + shape = (2, 3, 4) + linear_idx = [19, 17, 0, 23] + desired = [[1, 1, 3], [1, 1, 1], [0, 0, 0], [1, 2, 3]] + actual = utils.unravel_index(linear_idx, shape) + self.assertAllEqual(desired, actual.eval()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/build/lib/t3f/variables.py b/build/lib/t3f/variables.py new file mode 100644 index 00000000..3c9cf569 --- /dev/null +++ b/build/lib/t3f/variables.py @@ -0,0 +1,141 @@ +import tensorflow as tf + +from t3f.tensor_train import TensorTrain +from t3f.tensor_train_batch import TensorTrainBatch + +eager_mode_supported = False +try: + from tensorflow.python.eager import context + eager_mode_supported = True +except ImportError: + pass + + +def get_variable(name, + dtype=None, + initializer=None, + regularizer=None, + trainable=True, + collections=None, + caching_device=None, + validate_shape=True): + """Returns TensorTrain object with tf.Variables as the TT-cores. + + Args: + name: The name of the new or existing TensorTrain variable. + Used to name the TT-cores. + dtype: Type of the new or existing TensorTrain variable TT-cores (defaults + to DT_FLOAT). + initializer: TensorTrain or TensorTrainBatch, initializer for the variable + if one is created. + regularizer: A (TensorTrain -> Tensor or None) function; the result of + applying it on a newly created variable will be added to the collection + GraphKeys.REGULARIZATION_LOSSES and can be used for regularization. + trainable: If True also add the variable to the graph collection + GraphKeys.TRAINABLE_VARIABLES (see tf.Variable). + collections: List of graph collections keys to add the Variables + (underlying TT-cores). Defaults to [GraphKeys.GLOBAL_VARIABLES] + (see tf.Variable). + caching_device: Optional device string or function describing where + the Variable should be cached for reading. Defaults to the Variable's + device. If not None, caches on another device. Typical use is to cache + on the device where the Ops using the Variable reside, to deduplicate + copying through Switch and other conditional statements. + validate_shape: If False, allows the variable to be initialized with a value + of unknown shape. If True, the default, the shape of initial_value must be + known. + + Returns: + The created or existing `TensorTrain` object with tf.Variables TT-cores. + + Raises: + `ValueError`: when creating a new variable and shape is not declared, when + violating reuse during variable creation, or when initializer dtype and + dtype don't match. Reuse is set inside variable_scope. + """ + # TODO: support validate shape: check that the tensor dimensions are correct, + # but ignore the ranks. + # TODO: add validate ranks flag. + + reuse = tf.get_variable_scope().reuse + if not reuse and initializer is None: + raise ValueError('Scope reuse is False and initializer is not provided.') + + variable_cores = [] + in_eager_mode = False + if eager_mode_supported: + if context.in_eager_mode(): + in_eager_mode = True + + if reuse and not in_eager_mode: + # Find an existing variable in the collection. + path = tf.get_variable_scope().name + if path != '' and path[-1] != '/': + path += '/' + path += name + + found_v = None + for v in tf.get_collection('TensorTrainVariables'): + if v.name == path: + found_v = v + break + if found_v is None: + raise ValueError('ValueError: Variable %s does not exist, or was not ' + 'created with t3f.get_tt_variable(). Did you mean to ' + 'set reuse=None in VarScope?' % name) + with tf.variable_scope(name): + # Try to get the first core through tf.get_variable to check that we don't + # violate reuse: it will raise a ValueError otherwise. + tf.get_variable('core_0') + return found_v + else: + # Create new variable. + with tf.variable_scope(name): + num_dims = initializer.ndims() + for i in range(num_dims): + curr_core_var = tf.get_variable('core_%d' % i, + initializer=initializer.tt_cores[i], + dtype=dtype, trainable=trainable, + collections=collections, + caching_device=caching_device) + variable_cores.append(curr_core_var) + if isinstance(initializer, TensorTrain): + v = TensorTrain(variable_cores, initializer.get_raw_shape(), + initializer.get_tt_ranks(), + convert_to_tensors=False) + else: + v = TensorTrainBatch(variable_cores, initializer.get_raw_shape(), + initializer.get_tt_ranks(), initializer.batch_size, + convert_to_tensors=False) + + # Add the create TensorTrain object into a collection so that we can + # retrieve it in the future by get_tt_variable('name'). + tf.add_to_collection('TensorTrainVariables', v) + + # Run the regularizer if requested and save the resulting loss. + if regularizer: + with tf.name_scope(name + "/Regularizer/"): + loss = regularizer(v) + if loss is not None: + tf.logging.vlog(1, "Applied regularizer to %s and added the result %s " + "to REGULARIZATION_LOSSES.", v.name, loss.name) + tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, loss) + return v + + +def assign(ref, value, validate_shape=None, use_locking=None, name=None): + new_cores = [] + if name is None: + name = '' + with tf.variable_scope(name): + for i in range(ref.ndims()): + new_cores.append(tf.assign(ref.tt_cores[i], value.tt_cores[i], + use_locking=use_locking)) + if isinstance(value, TensorTrainBatch): + return TensorTrainBatch(new_cores, value.get_raw_shape(), + value.get_tt_ranks(), value.batch_size, + convert_to_tensors=False) + else: + return TensorTrain(new_cores, value.get_raw_shape(), + value.get_tt_ranks(), + convert_to_tensors=False) diff --git a/build/lib/t3f/variables_test.py b/build/lib/t3f/variables_test.py new file mode 100644 index 00000000..de96a22d --- /dev/null +++ b/build/lib/t3f/variables_test.py @@ -0,0 +1,82 @@ +import numpy as np +import tensorflow as tf + +from t3f import variables +from t3f import ops +from t3f import initializers + +class VariablesTest(tf.test.TestCase): + + def testGetExistingVariable(self): + init = initializers.random_tensor([2, 3, 2], tt_rank=2) + tt_1 = variables.get_variable('tt_1', initializer=init) + with tf.variable_scope('test'): + tt_2 = variables.get_variable('tt_2', initializer=init) + with self.test_session(): + tf.global_variables_initializer().run() + with self.assertRaises(ValueError): + # The variable already exists and scope.reuse is False by default. + variables.get_variable('tt_1') + with self.assertRaises(ValueError): + with tf.variable_scope('', reuse=True): + # The variable doesn't exist. + variables.get_variable('tt_3') + + with tf.variable_scope('', reuse=True): + tt_1_copy = variables.get_variable('tt_1') + self.assertAllClose(ops.full(tt_1).eval(), ops.full(tt_1_copy).eval()) + + with tf.variable_scope('', reuse=True): + # Again try to retrieve an existing variable, but pass an initializer + # and check that it still works. + tt_1_copy = variables.get_variable('tt_1', initializer=0 * init) + self.assertAllClose(ops.full(tt_1).eval(), ops.full(tt_1_copy).eval()) + + with self.assertRaises(ValueError): + with tf.variable_scope('', reuse=True): + # The variable is defined in a different scope + variables.get_variable('tt_2') + + with self.assertRaises(ValueError): + with tf.variable_scope('nottest', reuse=True): + # The variable is defined in a different scope + variables.get_variable('tt_2') + + with tf.variable_scope('test', reuse=True): + tt_2_copy = variables.get_variable('tt_2') + self.assertAllClose(ops.full(tt_2).eval(), ops.full(tt_2_copy).eval()) + + def testAttributes(self): + # Test that after converting an initializer into a variable all the + # attributes stays the same. + tens = initializers.random_tensor([2, 3, 2], tt_rank=2) + tens_v = variables.get_variable('tt_tens', initializer=tens) + mat = initializers.random_matrix([[3, 2, 2], [3, 3, 3]], tt_rank=3) + mat_v = variables.get_variable('tt_mat', initializer=mat) + for (init, var) in [[tens, tens_v], [mat, mat_v]]: + self.assertEqual(init.get_shape(), var.get_shape()) + self.assertEqual(init.get_raw_shape(), var.get_raw_shape()) + self.assertEqual(init.ndims(), var.ndims()) + self.assertEqual(init.get_tt_ranks(), var.get_tt_ranks()) + self.assertEqual(init.is_tt_matrix(), var.is_tt_matrix()) + + def testAssign(self): + old_init = initializers.random_tensor([2, 3, 2], tt_rank=2) + tt = variables.get_variable('tt', initializer=old_init) + new_init = initializers.random_tensor([2, 3, 2], tt_rank=2) + assigner = variables.assign(tt, new_init) + with self.test_session(): + tf.global_variables_initializer().run() + init_value = ops.full(tt).eval() + assigner_value = ops.full(assigner).eval() + after_value = ops.full(tt) + after_value = after_value.eval() + self.assertAllClose(assigner_value, after_value) + # Assert that the value actually changed: + abs_diff = np.linalg.norm((init_value - after_value).flatten()) + rel_diff = abs_diff / np.linalg.norm((init_value).flatten()) + self.assertGreater(rel_diff, 0.2) + + +if __name__ == "__main__": + tf.test.main() diff --git a/t3f/variables.py b/t3f/variables.py index e391ad67..3c9cf569 100644 --- a/t3f/variables.py +++ b/t3f/variables.py @@ -3,6 +3,13 @@ from t3f.tensor_train import TensorTrain from t3f.tensor_train_batch import TensorTrainBatch +eager_mode_supported = False +try: + from tensorflow.python.eager import context + eager_mode_supported = True +except ImportError: + pass + def get_variable(name, dtype=None, @@ -55,7 +62,12 @@ def get_variable(name, raise ValueError('Scope reuse is False and initializer is not provided.') variable_cores = [] - if reuse: + in_eager_mode = False + if eager_mode_supported: + if context.in_eager_mode(): + in_eager_mode = True + + if reuse and not in_eager_mode: # Find an existing variable in the collection. path = tf.get_variable_scope().name if path != '' and path[-1] != '/': From bc421a3263bb969b755365356816e1ab26312928 Mon Sep 17 00:00:00 2001 From: geom-score Date: Mon, 5 Mar 2018 16:55:32 +0300 Subject: [PATCH 005/233] removed installation files --- build/lib/t3f/__init__.py | 81 -- build/lib/t3f/approximate.py | 167 --- build/lib/t3f/approximate_test.py | 101 -- build/lib/t3f/batch_ops.py | 218 ---- build/lib/t3f/batch_ops_test.py | 161 --- build/lib/t3f/decompositions.py | 599 ----------- build/lib/t3f/decompositions_test.py | 176 ---- build/lib/t3f/examples_tests.py | 43 - build/lib/t3f/initializers.py | 789 -------------- build/lib/t3f/initializers_test.py | 176 ---- build/lib/t3f/kronecker.py | 237 ----- build/lib/t3f/kronecker_test.py | 182 ---- build/lib/t3f/ops.py | 1204 ---------------------- build/lib/t3f/ops_test.py | 880 ---------------- build/lib/t3f/regularizers.py | 79 -- build/lib/t3f/riemannian.py | 751 -------------- build/lib/t3f/riemannian_test.py | 250 ----- build/lib/t3f/shapes.py | 281 ----- build/lib/t3f/tensor_train.py | 229 ---- build/lib/t3f/tensor_train_base.py | 189 ---- build/lib/t3f/tensor_train_batch.py | 367 ------- build/lib/t3f/tensor_train_batch_test.py | 77 -- build/lib/t3f/tensor_train_test.py | 138 --- build/lib/t3f/utils.py | 39 - build/lib/t3f/utils_test.py | 26 - build/lib/t3f/variables.py | 141 --- build/lib/t3f/variables_test.py | 82 -- 27 files changed, 7663 deletions(-) delete mode 100644 build/lib/t3f/__init__.py delete mode 100644 build/lib/t3f/approximate.py delete mode 100644 build/lib/t3f/approximate_test.py delete mode 100644 build/lib/t3f/batch_ops.py delete mode 100644 build/lib/t3f/batch_ops_test.py delete mode 100644 build/lib/t3f/decompositions.py delete mode 100644 build/lib/t3f/decompositions_test.py delete mode 100644 build/lib/t3f/examples_tests.py delete mode 100644 build/lib/t3f/initializers.py delete mode 100644 build/lib/t3f/initializers_test.py delete mode 100644 build/lib/t3f/kronecker.py delete mode 100644 build/lib/t3f/kronecker_test.py delete mode 100644 build/lib/t3f/ops.py delete mode 100644 build/lib/t3f/ops_test.py delete mode 100644 build/lib/t3f/regularizers.py delete mode 100644 build/lib/t3f/riemannian.py delete mode 100644 build/lib/t3f/riemannian_test.py delete mode 100644 build/lib/t3f/shapes.py delete mode 100644 build/lib/t3f/tensor_train.py delete mode 100644 build/lib/t3f/tensor_train_base.py delete mode 100644 build/lib/t3f/tensor_train_batch.py delete mode 100644 build/lib/t3f/tensor_train_batch_test.py delete mode 100644 build/lib/t3f/tensor_train_test.py delete mode 100644 build/lib/t3f/utils.py delete mode 100644 build/lib/t3f/utils_test.py delete mode 100644 build/lib/t3f/variables.py delete mode 100644 build/lib/t3f/variables_test.py diff --git a/build/lib/t3f/__init__.py b/build/lib/t3f/__init__.py deleted file mode 100644 index 60abb533..00000000 --- a/build/lib/t3f/__init__.py +++ /dev/null @@ -1,81 +0,0 @@ -from t3f.tensor_train_base import TensorTrainBase -from t3f.tensor_train import TensorTrain -from t3f.tensor_train_batch import TensorTrainBatch - -from t3f.variables import assign -from t3f.variables import get_variable - -from t3f.ops import add -from t3f.ops import cast -from t3f.ops import flat_inner -from t3f.ops import frobenius_norm -from t3f.ops import frobenius_norm_squared -from t3f.ops import full -from t3f.ops import matmul -from t3f.ops import multiply -from t3f.ops import quadratic_form -from t3f.ops import transpose -from t3f.ops import gather_nd -from t3f.ops import renormalize_tt_cores - -from t3f.batch_ops import concat_along_batch_dim -from t3f.batch_ops import gram_matrix -from t3f.batch_ops import multiply_along_batch_dim -from t3f.batch_ops import pairwise_flat_inner - -from t3f.initializers import matrix_with_random_cores -from t3f.initializers import matrix_batch_with_random_cores -from t3f.initializers import tensor_with_random_cores -from t3f.initializers import tensor_batch_with_random_cores -from t3f.initializers import random_tensor -from t3f.initializers import random_tensor_batch -from t3f.initializers import random_matrix -from t3f.initializers import random_matrix_batch -from t3f.initializers import tensor_ones -from t3f.initializers import tensor_zeros -from t3f.initializers import matrix_ones -from t3f.initializers import matrix_zeros -from t3f.initializers import eye -from t3f.initializers import ones_like -from t3f.initializers import zeros_like -from t3f.initializers import glorot_initializer -from t3f.initializers import he_initializer -from t3f.initializers import lecun_initializer - -from t3f.regularizers import cores_regularizer -from t3f.regularizers import l2_regularizer - -from t3f.riemannian import add_n_projected -from t3f.riemannian import pairwise_flat_inner_projected -from t3f.riemannian import project -from t3f.riemannian import project_matmul -from t3f.riemannian import project_sum - -from t3f.shapes import batch_size -from t3f.shapes import clean_raw_shape -from t3f.shapes import expand_batch_dim -from t3f.shapes import is_batch_broadcasting_possible -from t3f.shapes import lazy_batch_size -from t3f.shapes import lazy_raw_shape -from t3f.shapes import lazy_shape -from t3f.shapes import lazy_tt_ranks -from t3f.shapes import raw_shape -from t3f.shapes import shape -from t3f.shapes import squeeze_batch_dim -from t3f.shapes import tt_ranks - -from t3f.decompositions import orthogonalize_tt_cores -from t3f.decompositions import round -from t3f.decompositions import to_tt_matrix -from t3f.decompositions import to_tt_tensor - -import t3f.approximate as approximate -import t3f.kronecker as kronecker -import t3f.utils as utils - -_directly_imported = ['tensor_train_base', 'tensor_train', 'tensor_train_batch', - 'variables', 'ops', 'batch_ops', 'initializers', - 'regularizers', 'riemannian', 'shapes', 'decompositions'] - -__all__ = [s for s in dir() if - s not in _directly_imported and not s.startswith('_')] diff --git a/build/lib/t3f/approximate.py b/build/lib/t3f/approximate.py deleted file mode 100644 index 4cb8f495..00000000 --- a/build/lib/t3f/approximate.py +++ /dev/null @@ -1,167 +0,0 @@ -import numpy as np -import tensorflow as tf -from t3f.tensor_train_batch import TensorTrainBatch -from t3f import decompositions -from t3f import batch_ops - - -def add_n(tt_objects, max_tt_rank): - """Adds a bunch of TT-object and round after each summation. - - This version implements a slow-to-compile but fast-to-execute (at least on - a GPU) version: summing in a binary tree order. - I.e. it uses the following idea: - round(a + b + c + d) ~= round(round(a + b) + round(c + d)) - and so is able to compute the answer in log(N) parallel adds/rounds. - - Args: - tt_objects: a list of `TensorTrainBase` objects. - max_tt_rank: a number, TT-rank for each individual rounding. - - Returns: - Object of the same type as each input. - - See Also: - t3f.approximate.reduce_sum_batch - """ - prev_level = tt_objects - while len(prev_level) > 1: - next_level = [] - for i in range(0, len(prev_level), 2): - curr = prev_level[i] - if i + 1 < len(prev_level): - curr = decompositions.round(curr + prev_level[i + 1], max_tt_rank) - next_level.append(curr) - prev_level = next_level - return prev_level[0] - - -def reduce_sum_batch(tt_batch, max_tt_rank, coef=None): - """Sum of all TT-objects in the batch with rounding after each summation. - - This version implements a slow-to-compile but fast-to-execute (at least on - a GPU) version: summing in a binary tree order. - I.e. it uses the following idea: - round(a + b + c + d) ~= round(round(a + b) + round(c + d)) - and so is able to compute the answer in log(batch_size) parallel adds/rounds. - - Args: - tt_batch: `TensorTrainBatch` object. - max_tt_rank: a number, TT-rank for each individual rounding. - coef: tf.Tensor, its shape is either batch_size, or batch_size x N. - If coef is a vecotor of size batch_size, the result will - be (approximate) weighted sum. - If coef is a matrix of shape batch_size x N, the result will be - a `TensorTrainBatch` res containing N TT-object such that - res[j] ~= sum_i tt_batch[i] coef[i, j] - - Returns: - If coefficients are absent or is a vector of numbers, returns - a `TensorTrain` object representing (approximate) element-wise sum of all - the objects in the batch, weighted if coef is provided. - If coefficients is a matrix, returns `TensorTrainBatch`. - - See Also: - t3f.approximate.add_n - """ - ndims = tt_batch.ndims() - left_tt_rank_dim = tt_batch.left_tt_rank_dim - right_tt_rank_dim = tt_batch.right_tt_rank_dim - shape = tt_batch.get_raw_shape() - dtype = tt_batch.dtype - - is_batch_output = False - if coef is not None: - coef = tf.convert_to_tensor(coef) - if len(coef.get_shape()) == 1: - tt_batch = batch_ops.multiply_along_batch_dim(tt_batch, coef) - elif len(coef.get_shape()) == 2: - is_batch_output = True - output_size = coef.get_shape().as_list()[1] - # Coef is of size batch_size x N, need to duplicate the batch - # dimension xN. - if coef.shape[0] != tt_batch.batch_size: - raise ValueError('If coef is a matrix, it should be of shape ' - 'batch_size x N, got %d x %d instead ' - '(batch size is %d).' % (coef.shape[0], coef.shape[1], - tt_batch.batch_size)) - tt_batch_cores = [] - for core_idx in range(ndims): - curr_core = tt_batch.tt_cores[core_idx] - curr_shape = curr_core.get_shape().as_list() - new_shape = np.insert(curr_shape, 1, 1) - tiling = np.ones(len(new_shape)) - tiling[1] = output_size - curr_core = tf.tile(tf.reshape(curr_core, new_shape), tiling) - if core_idx == 0: - # Multiply the first TT-core by the provided coefficients. - # TODO: add t3f.utils.expands_dims_like(coef, curr_core) - shaped_coef = coef - for _ in range(len(curr_core.get_shape()) - len(coef.shape)): - shaped_coef = tf.expand_dims(shaped_coef, -1) - curr_core = curr_core * shaped_coef - # Merge the first two dimensions back into one. - raveled_shape = np.array(curr_shape).copy() - raveled_shape[0] *= output_size - curr_core = tf.reshape(curr_core, raveled_shape) - tt_batch_cores.append(curr_core) - tt_batch = TensorTrainBatch(tt_batch_cores, shape, - tt_batch.get_tt_ranks()) - - else: - raise ValueError('Coef cannot be more than 2-d.') - - if not is_batch_output: - output_size = 1 - - prev_level = tt_batch - while prev_level.batch_size > output_size: - current_level_cores = [] - for core_idx in range(ndims): - curr_orig_core = prev_level.tt_cores[core_idx] - if is_batch_output: - # Split the first dimension into batch_size x N - unraveled_shape = curr_orig_core.get_shape().as_list() - unraveled_shape = np.array(unraveled_shape).copy() - unraveled_shape[0] /= output_size - unraveled_shape = np.insert(unraveled_shape, 1, output_size) - curr_orig_core = tf.reshape(curr_orig_core, unraveled_shape) - - a_core = curr_orig_core[::2] - b_core = curr_orig_core[1::2] - - if a_core.get_shape()[0] > b_core.get_shape()[0]: - # Odd number of elements in the batch, will have to add dummy - # TT-object with the tt-cores filled with zeros. - zeros_shape = b_core.get_shape().as_list() - zeros_shape[0] = 1 - zeros = tf.zeros(zeros_shape, dtype) - b_core = tf.concat((b_core, zeros), axis=0) - - if is_batch_output: - # Merge the first two dimensions back into one. - a_core_shape = a_core.get_shape().as_list() - a_core_shape[0] = a_core_shape[0] * a_core_shape[1] - a_core_shape = np.delete(a_core_shape, 1) - a_core = tf.reshape(a_core, a_core_shape) - b_core_shape = b_core.get_shape().as_list() - b_core_shape[0] = b_core_shape[0] * b_core_shape[1] - b_core_shape = np.delete(b_core_shape, 1) - b_core = tf.reshape(b_core, b_core_shape) - - if core_idx == 0: - curr_sum_core = tf.concat((a_core, b_core), axis=right_tt_rank_dim) - elif core_idx == ndims - 1: - curr_sum_core = tf.concat((a_core, b_core), axis=left_tt_rank_dim) - else: - zeros = tf.zeros(b_core.get_shape(), dtype) - upper = tf.concat((a_core, zeros), axis=right_tt_rank_dim) - lower = tf.concat((zeros, b_core), axis=right_tt_rank_dim) - curr_sum_core = tf.concat((upper, lower), axis=left_tt_rank_dim) - current_level_cores.append(curr_sum_core) - current_level = TensorTrainBatch(current_level_cores, shape) - prev_level = decompositions.round(current_level, max_tt_rank) - if is_batch_output: - return prev_level - else: - return prev_level[0] diff --git a/build/lib/t3f/approximate_test.py b/build/lib/t3f/approximate_test.py deleted file mode 100644 index 52a99320..00000000 --- a/build/lib/t3f/approximate_test.py +++ /dev/null @@ -1,101 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f import ops -from t3f import approximate -from t3f import initializers - - -class ApproximateTest(tf.test.TestCase): - - def testAddN(self): - # Sum a bunch of TT-matrices. - tt_a = initializers.random_matrix(((2, 1, 4), (2, 2, 2)), tt_rank=2) - tt_b = initializers.random_matrix(((2, 1, 4), (2, 2, 2)), - tt_rank=[1, 2, 4, 1]) - - def desired(tt_objects): - res = tt_objects[0] - for tt in tt_objects[1:]: - res += tt - return res - - with self.test_session() as sess: - res_actual = ops.full(approximate.add_n([tt_a, tt_b], 6)) - res_desired = ops.full(desired([tt_a, tt_b])) - res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) - self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) - - res_actual = ops.full(approximate.add_n([tt_a, tt_b, tt_a], 8)) - res_desired = ops.full(desired([tt_a, tt_b, tt_a])) - res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) - self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) - - res_actual = ops.full(approximate.add_n([tt_a, tt_b, tt_a, tt_a, tt_a], 12)) - res_desired = ops.full(desired([tt_a, tt_b, tt_a, tt_a, tt_a])) - res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) - self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) - - def testReduceSumBatch(self): - # Sum a batch of TT-tensors. - - def desired(tt_batch): - res = tt_batch[0] - for i in range(1, tt_batch.batch_size): - res += tt_batch[i] - return res - for batch_size in [2, 3, 4, 5]: - with self.test_session() as sess: - tt_batch = initializers.random_tensor_batch((4, 3, 5), - tt_rank=2, - batch_size=batch_size) - res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 10)) - res_desired = ops.full(desired(tt_batch)) - res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) - self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) - - def testReduceSumBatchWeighted(self): - # Weighted sum of a batch of TT-tensors. - - def desired(tt_batch, coef): - res = coef[0] * tt_batch[0] - for i in range(1, tt_batch.batch_size): - res += coef[i] * tt_batch[i] - return res - with self.test_session() as sess: - tt_batch = initializers.random_tensor_batch((4, 3, 5), - tt_rank=3, - batch_size=3) - res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 9, - [1.2, -0.2, 1])) - res_desired = ops.full(desired(tt_batch, [1.2, -0.2, 1])) - res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) - self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) - - def testReduceSumBatchMultipleWeighted(self): - # Multiple weighted sums of a batch of TT-tensors. - - def desired(tt_batch, coef): - res = coef[0] * tt_batch[0] - for i in range(1, tt_batch.batch_size): - res += coef[i] * tt_batch[i] - return res - with self.test_session() as sess: - tt_batch = initializers.random_tensor_batch((4, 3, 5), - tt_rank=2, - batch_size=3) - coef = [[1., 0.1], - [0.9, -0.2], - [0.3, 0.3]] - coef = np.array(coef).astype(np.float32) - res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 6, - coef)) - res_desired_1 = ops.full(desired(tt_batch, coef[:, 0])) - res_desired_2 = ops.full(desired(tt_batch, coef[:, 1])) - res_desired = tf.stack((res_desired_1, res_desired_2)) - res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) - self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) - - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/batch_ops.py b/build/lib/t3f/batch_ops.py deleted file mode 100644 index bd89bb36..00000000 --- a/build/lib/t3f/batch_ops.py +++ /dev/null @@ -1,218 +0,0 @@ -import tensorflow as tf - -from t3f.tensor_train_base import TensorTrainBase -from t3f.tensor_train_batch import TensorTrainBatch -from t3f import ops - - -def concat_along_batch_dim(tt_list): - """Concat all TensorTrainBatch objects along batch dimension. - - Args: - tt_list: a list of TensorTrainBatch objects. - - Returns: - TensorTrainBatch - """ - ndims = tt_list[0].ndims() - - if isinstance(tt_list, TensorTrainBase): - # Not a list but just one element, nothing to concat. - return tt_list - - for batch_idx in range(len(tt_list)): - if not isinstance(tt_list[batch_idx], TensorTrainBatch): - raise ValueError('All objects in the list should be TTBatch objects, got ' - '%s' % tt_list[batch_idx]) - for batch_idx in range(1, len(tt_list)): - if tt_list[batch_idx].get_raw_shape() != tt_list[0].get_raw_shape(): - raise ValueError('Shapes of all TT-batch objects should coincide, got %s ' - 'and %s' % (tt_list[0].get_raw_shape(), - tt_list[batch_idx].get_raw_shape())) - if tt_list[batch_idx].get_tt_ranks() != tt_list[0].get_tt_ranks(): - raise ValueError('TT-ranks of all TT-batch objects should coincide, got ' - '%s and %s' % (tt_list[0].get_tt_ranks(), - tt_list[batch_idx].get_tt_ranks())) - - res_cores = [] - for core_idx in range(ndims): - curr_core = tf.concat([tt.tt_cores[core_idx] for tt in tt_list], axis=0) - res_cores.append(curr_core) - - try: - batch_size = sum([tt.batch_size for tt in tt_list]) - except TypeError: - # The batch sizes are not defined and you can't sum Nones. - batch_size = None - - return TensorTrainBatch(res_cores, tt_list[0].get_raw_shape(), - tt_list[0].get_tt_ranks(), batch_size) - - -def multiply_along_batch_dim(batch_tt, weights): - """Multiply each TensorTrain in a batch by a number. - - Args: - batch_tt: TensorTrainBatch object, TT-matrices or TT-tensors. - weights: 1-D tf.Tensor (or something convertible to it like np.array) of size - tt.batch_size with weights. - - Returns: - TensorTrainBatch - """ - weights = tf.convert_to_tensor(weights) - tt_cores = list(batch_tt.tt_cores) - if batch_tt.is_tt_matrix(): - weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis, tf.newaxis] - else: - weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis] - tt_cores[0] = weights * tt_cores[0] - out_shape = batch_tt.get_raw_shape() - out_ranks = batch_tt.get_tt_ranks() - out_batch_size = batch_tt.batch_size - return TensorTrainBatch(tt_cores, out_shape, out_ranks, out_batch_size) - - -def gram_matrix(tt_vectors, matrix=None): - """Computes Gramian matrix of a batch of TT-vectors. - - If matrix is None, computes - res[i, j] = t3f.flat_inner(tt_vectors[i], tt_vectors[j]). - If matrix is present, computes - res[i, j] = t3f.flat_inner(tt_vectors[i], t3f.matmul(matrix, tt_vectors[j])) - or more shortly - res[i, j] = tt_vectors[i]^T * matrix * tt_vectors[j] - but is more efficient. - - Args: - tt_vectors: TensorTrainBatch. - matrix: None, or TensorTrain matrix. - - Returns: - tf.tensor with the Gram matrix. - - Complexity: - If the matrix is not present, the complexity is O(batch_size^2 d r^3 n) - where d is the number of - TT-cores (tt_vectors.ndims()), r is the largest TT-rank - max(tt_vectors.get_tt_rank()) - and n is the size of the axis dimension, e.g. - for a tensor of size 4 x 4 x 4, n is 4; - for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 - If the matrix of TT-rank R is present, the complexity is - O(batch_size^2 d R r^2 n (r + nR)) - where the matrix is of raw-shape (n, n, ..., n) x (n, n, ..., n); - r is the TT-rank of vectors tt_vectors; - R is the TT-rank of the matrix. - """ - return pairwise_flat_inner(tt_vectors, tt_vectors, matrix) - - -def pairwise_flat_inner(tt_1, tt_2, matrix=None): - """Computes all scalar products between two batches of TT-objects. - - If matrix is None, computes - res[i, j] = t3f.flat_inner(tt_1[i], tt_2[j]). - - If matrix is present, computes - res[i, j] = t3f.flat_inner(tt_1[i], t3f.matmul(matrix, tt_2[j])) - or more shortly - res[i, j] = tt_1[i]^T * matrix * tt_2[j] - but is more efficient. - - Args: - tt_1: TensorTrainBatch. - tt_2: TensorTrainBatch. - matrix: None, or TensorTrain matrix. - - Returns: - tf.tensor with the matrix of pairwise scalar products (flat inners). - - Complexity: - If the matrix is not present, the complexity is O(batch_size^2 d r^3 n) - where d is the number of - TT-cores (tt_vectors.ndims()), r is the largest TT-rank - max(tt_vectors.get_tt_rank()) - and n is the size of the axis dimension, e.g. - for a tensor of size 4 x 4 x 4, n is 4; - for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 - A more precise complexity is - O(batch_size^2 d r1 r2 n max(r1, r2)) - where r1 is the largest TT-rank of tt_a - and r2 is the largest TT-rank of tt_b. - If the matrix is present, the complexity is - O(batch_size^2 d R r1 r2 (n r1 + n m R + m r2)) - where - the matrix is of raw-shape (n, n, ..., n) x (m, m, ..., m) and TT-rank R; - tt_1 is of shape (n, n, ..., n) and is of the TT-rank r1; - tt_2 is of shape (m, m, ..., m) and is of the TT-rank r2; - """ - ndims = tt_1.ndims() - if matrix is None: - curr_core_1 = tt_1.tt_cores[0] - curr_core_2 = tt_2.tt_cores[0] - mode_string = 'ij' if tt_1.is_tt_matrix() else 'i' - einsum_str = 'pa{0}b,qc{0}d->pqbd'.format(mode_string) - res = tf.einsum(einsum_str, curr_core_1, curr_core_2) - for core_idx in range(1, ndims): - curr_core_1 = tt_1.tt_cores[core_idx] - curr_core_2 = tt_2.tt_cores[core_idx] - einsum_str = 'pqac,pa{0}b,qc{0}d->pqbd'.format(mode_string) - res = tf.einsum(einsum_str, res, curr_core_1, curr_core_2) - else: - # res[i, j] = tt_1[i] ^ T * matrix * tt_2[j] - if not tt_1.is_tt_matrix() or not tt_2.is_tt_matrix() or not matrix.is_tt_matrix(): - raise ValueError('When passing three arguments to pairwise_flat_inner, ' - 'the first 2 of them should be TT-vecors and the last ' - 'should be a TT-matrix. Got %s, %s, and %s instead.' % - (tt_1, tt_2, matrix)) - matrix_shape = matrix.get_raw_shape() - if not tt_1.get_raw_shape()[0].is_compatible_with(matrix_shape[0]): - raise ValueError('The shape of the first argument should be compatible ' - 'with the shape of the TT-matrix, that is it should be ' - 'possible to do the following matmul: ' - 'transpose(tt_1) * matrix. Got the first argument ' - '"%s" and matrix "%s"' % (tt_1, matrix)) - if not tt_2.get_raw_shape()[0].is_compatible_with(matrix_shape[1]): - raise ValueError('The shape of the second argument should be compatible ' - 'with the shape of the TT-matrix, that is it should be ' - 'possible to do the following matmul: ' - 'matrix * tt_2. Got the second argument ' - '"%s" and matrix "%s"' % (tt_2, matrix)) - - vectors_1_shape = tt_1.get_shape() - if vectors_1_shape[2] == 1 and vectors_1_shape[1] != 1: - # TODO: not very efficient, better to use different order in einsum. - tt_1 = ops.transpose(tt_1) - vectors_1_shape = tt_1.get_shape() - vectors_2_shape = tt_2.get_shape() - if vectors_2_shape[2] == 1 and vectors_2_shape[1] != 1: - # TODO: not very efficient, better to use different order in einsum. - tt_2 = ops.transpose(tt_2) - vectors_2_shape = tt_2.get_shape() - if vectors_1_shape[1] != 1: - # TODO: do something so that in case the shape is undefined on compilation - # it still works. - raise ValueError('The tt_vectors_1 argument should be vectors (not ' - 'matrices) with shape defined on compilation.') - if vectors_2_shape[1] != 1: - # TODO: do something so that in case the shape is undefined on compilation - # it still works. - raise ValueError('The tt_vectors_2 argument should be vectors (not ' - 'matrices) with shape defined on compilation.') - curr_core_1 = tt_1.tt_cores[0] - curr_core_2 = tt_2.tt_cores[0] - curr_matrix_core = matrix.tt_cores[0] - # We enumerate the dummy dimension (that takes 1 value) with `k`. - res = tf.einsum('pakib,cijd,qekjf->pqbdf', curr_core_1, curr_matrix_core, - curr_core_2) - for core_idx in range(1, ndims): - curr_core_1 = tt_1.tt_cores[core_idx] - curr_core_2 = tt_2.tt_cores[core_idx] - curr_matrix_core = matrix.tt_cores[core_idx] - res = tf.einsum('pqace,pakib,cijd,qekjf->pqbdf', res, curr_core_1, - curr_matrix_core, curr_core_2) - - # Squeeze to make the result of size batch_size x batch_size instead of - # batch_size x batch_size x 1 x 1. - return tf.squeeze(res) diff --git a/build/lib/t3f/batch_ops_test.py b/build/lib/t3f/batch_ops_test.py deleted file mode 100644 index 4c8cfd17..00000000 --- a/build/lib/t3f/batch_ops_test.py +++ /dev/null @@ -1,161 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f import ops -from t3f import batch_ops -from t3f import initializers - - -class BatchOpsTest(tf.test.TestCase): - - def testConcatMatrix(self): - # Test concating TTMatrix batches along batch dimension. - first = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=1) - second = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=4) - third = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3) - first_res = batch_ops.concat_along_batch_dim((first)) - first_res = ops.full(first_res) - first_second_res = batch_ops.concat_along_batch_dim((first, second)) - first_second_res = ops.full(first_second_res) - first_second_third_res = batch_ops.concat_along_batch_dim((first, second, - third)) - first_second_third_res = ops.full(first_second_third_res) - - first_full = ops.full(first) - second_full = ops.full(second) - third_full = ops.full(third) - first_desired = first_full - first_second_desired = tf.concat((first_full, second_full), axis=0) - first_second_third_desired = tf.concat((first_full, second_full, third_full), - axis=0) - with self.test_session() as sess: - res = sess.run((first_res, first_second_res, first_second_third_res, - first_desired, first_second_desired, - first_second_third_desired)) - first_res_val = res[0] - first_second_res_val = res[1] - first_second_third_res_val = res[2] - first_desired_val = res[3] - first_second_desired_val = res[4] - first_second_third_desired_val = res[5] - self.assertAllClose(first_res_val, first_desired_val) - self.assertAllClose(first_second_res_val, first_second_desired_val) - self.assertAllClose(first_second_third_res_val, first_second_third_desired_val) - - def testConcatTensorPlaceholders(self): - # Test concating TTTensors of unknown batch sizes along batch dimension. - number_of_objects = tf.placeholder(tf.int32) - all = initializers.random_tensor_batch((2, 3), batch_size=5) - actual = batch_ops.concat_along_batch_dim((all[:number_of_objects], - all[number_of_objects:])) - with self.test_session() as sess: - desired_val, actual_val = sess.run((ops.full(all), ops.full(actual)), - feed_dict={number_of_objects: 2}) - self.assertAllClose(desired_val, actual_val) - - def testConcatMatrixPlaceholders(self): - # Test concating TTMatrices of unknown batch sizes along batch dimension. - number_of_objects = tf.placeholder(tf.int32) - all = initializers.random_matrix_batch(((2, 3), (2, 3)), batch_size=5) - actual = batch_ops.concat_along_batch_dim((all[:number_of_objects], - all[number_of_objects:])) - with self.test_session() as sess: - desired_val, actual_val = sess.run((ops.full(all), ops.full(actual)), - feed_dict={number_of_objects: 2}) - self.assertAllClose(desired_val, actual_val) - - def testBatchMultiply(self): - # Test multiplying batch of TTMatrices by individual numbers. - tt = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3) - weights = [0.1, 0, -10] - actual = batch_ops.multiply_along_batch_dim(tt, weights) - individual_desired = [weights[i] * tt[i:i+1] for i in range(3)] - desired = batch_ops.concat_along_batch_dim(individual_desired) - with self.test_session() as sess: - desired_val, acutual_val = sess.run((ops.full(desired), ops.full(actual))) - self.assertAllClose(desired_val, acutual_val) - - def testGramMatrix(self): - # Test Gram Matrix of a batch of TT vectors. - tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=5) - res_actual = batch_ops.gram_matrix(tt_vectors) - full_vectors = tf.reshape(ops.full(tt_vectors), (5, 6)) - res_desired = tf.matmul(full_vectors, tf.transpose(full_vectors)) - res_desired = tf.squeeze(res_desired) - with self.test_session() as sess: - res_actual_val, res_desired_val = sess.run((res_actual, res_desired)) - self.assertAllClose(res_desired_val, res_actual_val) - - def testGramMatrixWithMatrix(self): - # Test Gram Matrix of a batch of TT vectors with providing a matrix, so we - # should compute - # res[i, j] = tt_vectors[i] ^ T * matrix * tt_vectors[j] - tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=4) - matrix = initializers.random_matrix(((2, 3), (2, 3))) - res_actual = batch_ops.gram_matrix(tt_vectors, matrix) - full_vectors = tf.reshape(ops.full(tt_vectors), (4, 6)) - with self.test_session() as sess: - res = sess.run((res_actual, full_vectors, ops.full(matrix))) - res_actual_val, vectors_val, matrix_val = res - res_desired_val = np.zeros((4, 4)) - for i in range(4): - for j in range(4): - curr_val = np.dot(vectors_val[i], matrix_val) - curr_val = np.dot(curr_val, vectors_val[j]) - res_desired_val[i, j] = curr_val - self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) - - def testPairwiseFlatInnerTensor(self): - # Test pairwise_flat_inner of a batch of TT tensors. - tt_tensors_1 = initializers.random_tensor_batch((2, 3, 2), batch_size=5) - tt_tensors_2 = initializers.random_tensor_batch((2, 3, 2), batch_size=5) - res_actual = batch_ops.pairwise_flat_inner(tt_tensors_1, tt_tensors_2) - full_tensors_1 = tf.reshape(ops.full(tt_tensors_1), (5, 12)) - full_tensors_2 = tf.reshape(ops.full(tt_tensors_2), (5, 12)) - res_desired = tf.matmul(full_tensors_1, tf.transpose(full_tensors_2)) - res_desired = tf.squeeze(res_desired) - with self.test_session() as sess: - res_actual_val, res_desired_val = sess.run((res_actual, res_desired)) - self.assertAllClose(res_desired_val, res_actual_val) - - def testPairwiseFlatInnerMatrix(self): - # Test pairwise_flat_inner of a batch of TT matrices. - tt_vectors_1 = initializers.random_matrix_batch(((2, 3), (2, 3)), - batch_size=5) - tt_vectors_2 = initializers.random_matrix_batch(((2, 3), (2, 3)), - batch_size=5) - res_actual = batch_ops.pairwise_flat_inner(tt_vectors_1, tt_vectors_2) - full_vectors_1 = tf.reshape(ops.full(tt_vectors_1), (5, 36)) - full_vectors_2 = tf.reshape(ops.full(tt_vectors_2), (5, 36)) - res_desired = tf.matmul(full_vectors_1, tf.transpose(full_vectors_2)) - res_desired = tf.squeeze(res_desired) - with self.test_session() as sess: - res_actual_val, res_desired_val = sess.run((res_actual, res_desired)) - self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) - - def testPairwiseFlatInnerVectorsWithMatrix(self): - # Test pairwise_flat_inner of a batch of TT vectors with providing a matrix, - # so we should compute - # res[i, j] = tt_vectors[i] ^ T * matrix * tt_vectors[j] - tt_vectors_1 = initializers.random_matrix_batch(((2, 3), None), batch_size=2) - tt_vectors_2 = initializers.random_matrix_batch(((2, 3), None), batch_size=3) - matrix = initializers.random_matrix(((2, 3), (2, 3))) - res_actual = batch_ops.pairwise_flat_inner(tt_vectors_1, tt_vectors_2, - matrix) - full_vectors_1 = tf.reshape(ops.full(tt_vectors_1), (2, 6)) - full_vectors_2 = tf.reshape(ops.full(tt_vectors_2), (3, 6)) - with self.test_session() as sess: - res = sess.run((res_actual, full_vectors_1, full_vectors_2, - ops.full(matrix))) - res_actual_val, vectors_1_val, vectors_2_val, matrix_val = res - res_desired_val = np.zeros((2, 3)) - for i in range(2): - for j in range(3): - curr_val = np.dot(vectors_1_val[i], matrix_val) - curr_val = np.dot(curr_val, vectors_2_val[j]) - res_desired_val[i, j] = curr_val - self.assertAllClose(res_desired_val, res_actual_val) - -if __name__ == "__main__": - tf.test.main() - diff --git a/build/lib/t3f/decompositions.py b/build/lib/t3f/decompositions.py deleted file mode 100644 index b16ee7f0..00000000 --- a/build/lib/t3f/decompositions.py +++ /dev/null @@ -1,599 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f.tensor_train import TensorTrain -from t3f.tensor_train_batch import TensorTrainBatch -from t3f import shapes - - -def to_tt_matrix(mat, shape, max_tt_rank=10, epsilon=None): - """Converts a given matrix or vector to a TT-matrix. - - The matrix dimensions should factorize into d numbers. - If e.g. the dimensions are prime numbers, it's usually better to - pad the matrix with zeros until the dimensions factorize into - (ideally) 3-8 numbers. - - Args: - mat: two dimensional tf.Tensor (a matrix). - shape: two dimensional array (np.array or list of lists) - Represents the tensor shape of the matrix. - E.g. for a (a1 * a2 * a3) x (b1 * b2 * b3) matrix `shape` should be - ((a1, a2, a3), (b1, b2, b3)) - `shape[0]`` and `shape[1]`` should have the same length. - For vectors you may use ((a1, a2, a3), (1, 1, 1)) or, equivalently, - ((a1, a2, a3), None) - max_tt_rank: a number or a list of numbers - If a number, than defines the maximal TT-rank of the result. - If a list of numbers, than `max_tt_rank` length should be d+1 - (where d is the length of `shape[0]`) and `max_tt_rank[i]` defines - the maximal (i+1)-th TT-rank of the result. - The following two versions are equivalent - `max_tt_rank = r` - and - `max_tt_rank = r * np.ones(d-1)` - epsilon: a floating point number or None - If the TT-ranks are not restricted (`max_tt_rank=np.inf`), then - the result would be guarantied to be `epsilon` close to `mat` - in terms of relative Frobenius error: - ||res - mat||_F / ||mat||_F <= epsilon - If the TT-ranks are restricted, providing a loose `epsilon` may reduce - the TT-ranks of the result. - E.g. - to_tt_matrix(mat, shape, max_tt_rank=100, epsilon=0.9) - will probably return you a TT-matrix with TT-ranks close to 1, not 100. - Note that providing a nontrivial (= not equal to None) `epsilon` will make - the TT-ranks of the result undefined on the compilation stage - (e.g. res.get_tt_ranks() will return None, but t3f.tt_ranks(res).eval() - will work). - - Returns: - `TensorTrain` object containing a TT-matrix. - - Raises: - ValueError if max_tt_rank is less than 0, if max_tt_rank is not a number and - not a vector of length d + 1 where d is the number of dimensions (rank) of - the input tensor, if epsilon is less than 0. - """ - mat = tf.convert_to_tensor(mat) - # In case the shape is immutable. - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1])).astype(int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0])).astype(int) - - shape = np.array(shape) - tens = tf.reshape(mat, shape.flatten()) - d = len(shape[0]) - # transpose_idx = 0, d, 1, d+1 ... - transpose_idx = np.arange(2 * d).reshape(2, d).T.flatten() - transpose_idx = transpose_idx.astype(int) - tens = tf.transpose(tens, transpose_idx) - new_shape = np.prod(shape, axis=0) - tens = tf.reshape(tens, new_shape) - tt_tens = to_tt_tensor(tens, max_tt_rank, epsilon) - tt_cores = [] - static_tt_ranks = tt_tens.get_tt_ranks() - dynamic_tt_ranks = shapes.tt_ranks(tt_tens) - for core_idx in range(d): - curr_core = tt_tens.tt_cores[core_idx] - curr_rank = static_tt_ranks[core_idx].value - if curr_rank is None: - curr_rank = dynamic_tt_ranks[core_idx] - next_rank = static_tt_ranks[core_idx + 1].value - if next_rank is None: - next_rank = dynamic_tt_ranks[core_idx + 1] - curr_core_new_shape = (curr_rank, shape[0, core_idx], - shape[1, core_idx], next_rank) - curr_core = tf.reshape(curr_core, curr_core_new_shape) - tt_cores.append(curr_core) - return TensorTrain(tt_cores, shape, tt_tens.get_tt_ranks()) - - -# TODO: implement epsilon. -def to_tt_tensor(tens, max_tt_rank=10, epsilon=None): - """Converts a given tf.Tensor to a TT-tensor of the same shape. - - Args: - tens: tf.Tensor - max_tt_rank: a number or a list of numbers - If a number, than defines the maximal TT-rank of the result. - If a list of numbers, than `max_tt_rank` length should be d+1 - (where d is the rank of `tens`) and `max_tt_rank[i]` defines - the maximal (i+1)-th TT-rank of the result. - The following two versions are equivalent - `max_tt_rank = r` - and - `max_tt_rank = r * np.ones(d-1)` - epsilon: a floating point number or None - If the TT-ranks are not restricted (`max_tt_rank=np.inf`), then - the result would be guarantied to be `epsilon` close to `tens` - in terms of relative Frobenius error: - ||res - tens||_F / ||tens||_F <= epsilon - If the TT-ranks are restricted, providing a loose `epsilon` may - reduce the TT-ranks of the result. - E.g. - to_tt_tensor(tens, max_tt_rank=100, epsilon=0.9) - will probably return you a TT-tensor with TT-ranks close to 1, not 100. - Note that providing a nontrivial (= not equal to None) `epsilon` will make - the TT-ranks of the result undefined on the compilation stage - (e.g. res.get_tt_ranks() will return None, but t3f.tt_ranks(res).eval() - will work). - - Returns: - `TensorTrain` object containing a TT-tensor. - - Raises: - ValueError if the rank (number of dimensions) of the input tensor is - not defined, if max_tt_rank is less than 0, if max_tt_rank is not a number - and not a vector of length d + 1 where d is the number of dimensions (rank) - of the input tensor, if epsilon is less than 0. - """ - tens = tf.convert_to_tensor(tens) - static_shape = tens.get_shape() - dynamic_shape = tf.shape(tens) - # Raises ValueError if ndims is not defined. - d = static_shape.__len__() - max_tt_rank = np.array(max_tt_rank).astype(np.int32) - if max_tt_rank < 1: - raise ValueError('Maximum TT-rank should be greater or equal to 1.') - if epsilon is not None and epsilon < 0: - raise ValueError('Epsilon should be non-negative.') - if max_tt_rank.size == 1: - max_tt_rank = (max_tt_rank * np.ones(d+1)).astype(np.int32) - elif max_tt_rank.size != d + 1: - raise ValueError('max_tt_rank should be a number or a vector of size (d+1) ' - 'where d is the number of dimensions (rank) of the tensor.') - ranks = [1] * (d + 1) - tt_cores = [] - are_tt_ranks_defined = True - for core_idx in range(d - 1): - curr_mode = static_shape[core_idx].value - if curr_mode is None: - curr_mode = dynamic_shape[core_idx] - rows = ranks[core_idx] * curr_mode - tens = tf.reshape(tens, [rows, -1]) - columns = tens.get_shape()[1].value - if columns is None: - columns = tf.shape(tens)[1] - s, u, v = tf.svd(tens, full_matrices=False) - if max_tt_rank[core_idx + 1] == 1: - ranks[core_idx + 1] = 1 - else: - try: - ranks[core_idx + 1] = min(max_tt_rank[core_idx + 1], rows, columns) - except TypeError: - # Some of the values are undefined on the compilation stage and thus - # they are tf.tensors instead of values. - min_dim = tf.minimum(rows, columns) - ranks[core_idx + 1] = tf.minimum(max_tt_rank[core_idx + 1], min_dim) - are_tt_ranks_defined = False - u = u[:, 0:ranks[core_idx + 1]] - s = s[0:ranks[core_idx + 1]] - v = v[:, 0:ranks[core_idx + 1]] - core_shape = (ranks[core_idx], curr_mode, ranks[core_idx + 1]) - tt_cores.append(tf.reshape(u, core_shape)) - tens = tf.matmul(tf.diag(s), tf.transpose(v)) - last_mode = static_shape[-1].value - if last_mode is None: - last_mode = dynamic_shape[-1] - core_shape = (ranks[d - 1], last_mode, ranks[d]) - tt_cores.append(tf.reshape(tens, core_shape)) - if not are_tt_ranks_defined: - ranks = None - return TensorTrain(tt_cores, static_shape, ranks) - - -# TODO: rename round so not to shadow python.round? -def round(tt, max_tt_rank=None, epsilon=None): - """TT-rounding procedure, returns a TT object with smaller TT-ranks. - - Args: - tt: `TensorTrain` object, TT-tensor or TT-matrix - max_tt_rank: a number or a list of numbers - If a number, than defines the maximal TT-rank of the result. - If a list of numbers, than `max_tt_rank` length should be d+1 - (where d is the rank of `tens`) and `max_tt_rank[i]` defines - the maximal (i+1)-th TT-rank of the result. - The following two versions are equivalent - `max_tt_rank = r` - and - `max_tt_rank = r * np.ones(d-1)` - epsilon: a floating point number or None - If the TT-ranks are not restricted (`max_tt_rank=np.inf`), then - the result would be guarantied to be `epsilon` close to `tt` - in terms of relative Frobenius error: - ||res - tt||_F / ||tt||_F <= epsilon - If the TT-ranks are restricted, providing a loose `epsilon` may - reduce the TT-ranks of the result. - E.g. - round(tt, max_tt_rank=100, epsilon=0.9) - will probably return you a TT-tensor with TT-ranks close to 1, not 100. - Note that providing a nontrivial (= not equal to None) `epsilon` will make - the TT-ranks of the result undefined on the compilation stage - (e.g. res.get_tt_ranks() will return None, but t3f.tt_ranks(res).eval() - will work). - - Returns: - `TensorTrain` object containing a TT-tensor. - - Raises: - ValueError if max_tt_rank is less than 0, if max_tt_rank is not a number and - not a vector of length d + 1 where d is the number of dimensions (rank) of - the input tensor, if epsilon is less than 0. - """ - if isinstance(tt, TensorTrainBatch): - return _round_batch_tt(tt, max_tt_rank, epsilon) - else: - return _round_tt(tt, max_tt_rank, epsilon) - - -def _round_tt(tt, max_tt_rank, epsilon): - """Internal function that rounds a TensorTrain (not batch). - - See t3f.round for details. - """ - ndims = tt.ndims() - max_tt_rank = np.array(max_tt_rank).astype(np.int32) - if max_tt_rank < 1: - raise ValueError('Maximum TT-rank should be greater or equal to 1.') - if epsilon is not None and epsilon < 0: - raise ValueError('Epsilon should be non-negative.') - if max_tt_rank.size == 1: - max_tt_rank = (max_tt_rank * np.ones(ndims + 1)).astype(np.int32) - elif max_tt_rank.size != ndims + 1: - raise ValueError('max_tt_rank should be a number or a vector of size (d+1) ' - 'where d is the number of dimensions (rank) of the tensor.') - raw_shape = shapes.lazy_raw_shape(tt) - - tt_cores = orthogonalize_tt_cores(tt).tt_cores - # Copy cores references so we can change the cores. - tt_cores = list(tt_cores) - - ranks = [1] * (ndims + 1) - are_tt_ranks_defined = True - # Right to left SVD compression. - for core_idx in range(ndims - 1, 0, -1): - curr_core = tt_cores[core_idx] - if tt.is_tt_matrix(): - curr_mode_left = raw_shape[0][core_idx] - curr_mode_right = raw_shape[1][core_idx] - curr_mode = curr_mode_left * curr_mode_right - else: - curr_mode = raw_shape[0][core_idx] - - columns = curr_mode * ranks[core_idx + 1] - curr_core = tf.reshape(curr_core, [-1, columns]) - rows = curr_core.get_shape()[0].value - if rows is None: - rows = tf.shape(curr_core)[0] - if max_tt_rank[core_idx] == 1: - ranks[core_idx] = 1 - else: - try: - ranks[core_idx] = min(max_tt_rank[core_idx], rows, columns) - except TypeError: - # Some of the values are undefined on the compilation stage and thus - # they are tf.tensors instead of values. - min_dim = tf.minimum(rows, columns) - ranks[core_idx] = tf.minimum(max_tt_rank[core_idx], min_dim) - are_tt_ranks_defined = False - s, u, v = tf.svd(curr_core, full_matrices=False) - u = u[:, 0:ranks[core_idx]] - s = s[0:ranks[core_idx]] - v = v[:, 0:ranks[core_idx]] - if tt.is_tt_matrix(): - core_shape = (ranks[core_idx], curr_mode_left, curr_mode_right, - ranks[core_idx + 1]) - else: - core_shape = (ranks[core_idx], curr_mode, ranks[core_idx + 1]) - tt_cores[core_idx] = tf.reshape(tf.transpose(v), core_shape) - prev_core_shape = (-1, rows) - tt_cores[core_idx - 1] = tf.reshape(tt_cores[core_idx - 1], prev_core_shape) - tt_cores[core_idx - 1] = tf.matmul(tt_cores[core_idx - 1], u) - tt_cores[core_idx - 1] = tf.matmul(tt_cores[core_idx - 1], tf.diag(s)) - - if tt.is_tt_matrix(): - core_shape = (ranks[0], raw_shape[0][0], raw_shape[1][0], ranks[1]) - else: - core_shape = (ranks[0], raw_shape[0][0], ranks[1]) - tt_cores[0] = tf.reshape(tt_cores[0], core_shape) - if not are_tt_ranks_defined: - ranks = None - return TensorTrain(tt_cores, tt.get_raw_shape(), ranks) - - -def _round_batch_tt(tt, max_tt_rank, epsilon): - """Internal function that rounds a TensorTrainBatch. - - See t3f.round for details. - """ - ndims = tt.ndims() - max_tt_rank = np.array(max_tt_rank).astype(np.int32) - if max_tt_rank < 1: - raise ValueError('Maximum TT-rank should be greater or equal to 1.') - if epsilon is not None and epsilon < 0: - raise ValueError('Epsilon should be non-negative.') - if max_tt_rank.size == 1: - max_tt_rank = (max_tt_rank * np.ones(ndims + 1)).astype(np.int32) - elif max_tt_rank.size != ndims + 1: - raise ValueError('max_tt_rank should be a number or a vector of size (d+1) ' - 'where d is the number of dimensions (rank) of the tensor.') - raw_shape = shapes.lazy_raw_shape(tt) - batch_size = shapes.lazy_batch_size(tt) - - tt_cores = orthogonalize_tt_cores(tt).tt_cores - # Copy cores references so we can change the cores. - tt_cores = list(tt_cores) - - ranks = [1] * (ndims + 1) - are_tt_ranks_defined = True - # Right to left SVD compression. - for core_idx in range(ndims - 1, 0, -1): - curr_core = tt_cores[core_idx] - if tt.is_tt_matrix(): - curr_mode_left = raw_shape[0][core_idx] - curr_mode_right = raw_shape[1][core_idx] - curr_mode = curr_mode_left * curr_mode_right - else: - curr_mode = raw_shape[0][core_idx] - - columns = curr_mode * ranks[core_idx + 1] - curr_core = tf.reshape(curr_core, (batch_size, -1, columns)) - rows = curr_core.get_shape()[1].value - if rows is None: - rows = tf.shape(curr_core)[1] - if max_tt_rank[core_idx] == 1: - ranks[core_idx] = 1 - else: - try: - ranks[core_idx] = min(max_tt_rank[core_idx], rows, columns) - except TypeError: - # Some of the values are undefined on the compilation stage and thus - # they are tf.tensors instead of values. - min_dim = tf.minimum(rows, columns) - ranks[core_idx] = tf.minimum(max_tt_rank[core_idx], min_dim) - are_tt_ranks_defined = False - s, u, v = tf.svd(curr_core, full_matrices=False) - u = u[:, :, 0:ranks[core_idx]] - s = s[:, 0:ranks[core_idx]] - v = v[:, :, 0:ranks[core_idx]] - if tt.is_tt_matrix(): - core_shape = (batch_size, ranks[core_idx], curr_mode_left, curr_mode_right, - ranks[core_idx + 1]) - else: - core_shape = (batch_size, ranks[core_idx], curr_mode, ranks[core_idx + 1]) - tt_cores[core_idx] = tf.reshape(tf.transpose(v, (0, 2, 1)), core_shape) - prev_core_shape = (batch_size, -1, rows) - tt_cores[core_idx - 1] = tf.reshape(tt_cores[core_idx - 1], prev_core_shape) - tt_cores[core_idx - 1] = tf.matmul(tt_cores[core_idx - 1], u) - tt_cores[core_idx - 1] = tf.matmul(tt_cores[core_idx - 1], tf.matrix_diag(s)) - - if tt.is_tt_matrix(): - core_shape = (batch_size, ranks[0], raw_shape[0][0], raw_shape[1][0], ranks[1]) - else: - core_shape = (batch_size, ranks[0], raw_shape[0][0], ranks[1]) - tt_cores[0] = tf.reshape(tt_cores[0], core_shape) - if not are_tt_ranks_defined: - ranks = None - return TensorTrainBatch(tt_cores, tt.get_raw_shape(), ranks, batch_size=tt.batch_size) - - -def orthogonalize_tt_cores(tt, left_to_right=True): - """Orthogonalize TT-cores of a TT-object. - - Args: - tt: TenosorTrain or a TensorTrainBatch. - left_to_right: bool, the direction of orthogonalization. - - Returns: - The same type as the input `tt` (TenosorTrain or a TensorTrainBatch). - """ - if isinstance(tt, TensorTrainBatch): - if left_to_right: - return _orthogonalize_batch_tt_cores_left_to_right(tt) - else: - raise NotImplementedError('Batch right to left orthogonalization is not ' - 'supported yet.') - else: - if left_to_right: - return _orthogonalize_tt_cores_left_to_right(tt) - else: - return _orthogonalize_tt_cores_right_to_left(tt) - - -def _orthogonalize_tt_cores_left_to_right(tt): - """Orthogonalize TT-cores of a TT-object in the left to right order. - Args: - tt: TenosorTrain or a TensorTrainBatch. - Returns: - The same type as the input `tt` (TenosorTrain or a TensorTrainBatch). - - Complexity: - for a single TT-object: - O(d r^3 n) - for a batch of TT-objects: - O(batch_size d r^3 n) - where - d is the number of TT-cores (tt.ndims()); - r is the largest TT-rank of tt max(tt.get_tt_rank()) - n is the size of the axis dimension, e.g. - for a tensor of size 4 x 4 x 4, n is 4; - for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 - """ - # Left to right orthogonalization. - ndims = tt.ndims() - raw_shape = shapes.lazy_raw_shape(tt) - tt_ranks = shapes.lazy_tt_ranks(tt) - next_rank = tt_ranks[0] - # Copy cores references so we can change the cores. - tt_cores = list(tt.tt_cores) - for core_idx in range(ndims - 1): - curr_core = tt_cores[core_idx] - # TT-ranks could have changed on the previous iteration, so `tt_ranks` can - # be outdated for the current TT-rank, but should be valid for the next - # TT-rank. - curr_rank = next_rank - next_rank = tt_ranks[core_idx + 1] - if tt.is_tt_matrix(): - curr_mode_left = raw_shape[0][core_idx] - curr_mode_right = raw_shape[1][core_idx] - curr_mode = curr_mode_left * curr_mode_right - else: - curr_mode = raw_shape[0][core_idx] - - qr_shape = (curr_rank * curr_mode, next_rank) - curr_core = tf.reshape(curr_core, qr_shape) - curr_core, triang = tf.qr(curr_core) - if triang.get_shape().is_fully_defined(): - triang_shape = triang.get_shape().as_list() - else: - triang_shape = tf.shape(triang) - # The TT-rank could have changed: if qr_shape is e.g. 4 x 10, than q would - # be of size 4 x 4 and r would be 4 x 10, which means that the next rank - # should be changed to 4. - next_rank = triang_shape[0] - if tt.is_tt_matrix(): - new_core_shape = (curr_rank, curr_mode_left, curr_mode_right, next_rank) - else: - new_core_shape = (curr_rank, curr_mode, next_rank) - tt_cores[core_idx] = tf.reshape(curr_core, new_core_shape) - - next_core = tf.reshape(tt_cores[core_idx + 1], (triang_shape[1], -1)) - tt_cores[core_idx + 1] = tf.matmul(triang, next_core) - - if tt.is_tt_matrix(): - last_core_shape = (next_rank, raw_shape[0][-1], raw_shape[1][-1], 1) - else: - last_core_shape = (next_rank, raw_shape[0][-1], 1) - tt_cores[-1] = tf.reshape(tt_cores[-1], last_core_shape) - # TODO: infer the tt_ranks. - return TensorTrain(tt_cores, tt.get_raw_shape()) - - -def _orthogonalize_batch_tt_cores_left_to_right(tt): - """Orthogonalize TT-cores of a batch TT-object in the left to right order. - - Args: - tt: TensorTrainBatch. - - Returns: - TensorTrainBatch - """ - # Left to right orthogonalization. - ndims = tt.ndims() - raw_shape = shapes.lazy_raw_shape(tt) - tt_ranks = shapes.lazy_tt_ranks(tt) - next_rank = tt_ranks[0] - batch_size = shapes.lazy_batch_size(tt) - - # Copy cores references so we can change the cores. - tt_cores = list(tt.tt_cores) - for core_idx in range(ndims - 1): - curr_core = tt_cores[core_idx] - # TT-ranks could have changed on the previous iteration, so `tt_ranks` can - # be outdated for the current TT-rank, but should be valid for the next - # TT-rank. - curr_rank = next_rank - next_rank = tt_ranks[core_idx + 1] - if tt.is_tt_matrix(): - curr_mode_left = raw_shape[0][core_idx] - curr_mode_right = raw_shape[1][core_idx] - curr_mode = curr_mode_left * curr_mode_right - else: - curr_mode = raw_shape[0][core_idx] - - qr_shape = (batch_size, curr_rank * curr_mode, next_rank) - curr_core = tf.reshape(curr_core, qr_shape) - curr_core, triang = tf.qr(curr_core) - if triang.get_shape().is_fully_defined(): - triang_shape = triang.get_shape().as_list() - else: - triang_shape = tf.shape(triang) - # The TT-rank could have changed: if qr_shape is e.g. 4 x 10, than q would - # be of size 4 x 4 and r would be 4 x 10, which means that the next rank - # should be changed to 4. - next_rank = triang_shape[1] - if tt.is_tt_matrix(): - new_core_shape = (batch_size, curr_rank, curr_mode_left, curr_mode_right, - next_rank) - else: - new_core_shape = (batch_size, curr_rank, curr_mode, next_rank) - - tt_cores[core_idx] = tf.reshape(curr_core, new_core_shape) - - next_core = tf.reshape(tt_cores[core_idx + 1], (batch_size, triang_shape[2], -1)) - tt_cores[core_idx + 1] = tf.matmul(triang, next_core) - - if tt.is_tt_matrix(): - last_core_shape = (batch_size, next_rank, raw_shape[0][-1], - raw_shape[1][-1], 1) - else: - last_core_shape = (batch_size, next_rank, raw_shape[0][-1], 1) - tt_cores[-1] = tf.reshape(tt_cores[-1], last_core_shape) - # TODO: infer the tt_ranks. - return TensorTrainBatch(tt_cores, tt.get_raw_shape(), batch_size=batch_size) - - -def _orthogonalize_tt_cores_right_to_left(tt): - """Orthogonalize TT-cores of a TT-object in the right to left order. - - Args: - tt: TenosorTrain or a TensorTrainBatch. - - Returns: - The same type as the input `tt` (TenosorTrain or a TensorTrainBatch). - """ - # Left to right orthogonalization. - ndims = tt.ndims() - raw_shape = shapes.lazy_raw_shape(tt) - tt_ranks = shapes.lazy_tt_ranks(tt) - prev_rank = tt_ranks[ndims] - # Copy cores references so we can change the cores. - tt_cores = list(tt.tt_cores) - for core_idx in range(ndims - 1, 0, -1): - curr_core = tt_cores[core_idx] - # TT-ranks could have changed on the previous iteration, so `tt_ranks` can - # be outdated for the current TT-rank, but should be valid for the next - # TT-rank. - curr_rank = prev_rank - prev_rank = tt_ranks[core_idx] - if tt.is_tt_matrix(): - curr_mode_left = raw_shape[0][core_idx] - curr_mode_right = raw_shape[1][core_idx] - curr_mode = curr_mode_left * curr_mode_right - else: - curr_mode = raw_shape[0][core_idx] - - qr_shape = (prev_rank, curr_mode * curr_rank) - curr_core = tf.reshape(curr_core, qr_shape) - curr_core, triang = tf.qr(tf.transpose(curr_core)) - curr_core = tf.transpose(curr_core) - triang = tf.transpose(triang) - if triang.get_shape().is_fully_defined(): - triang_shape = triang.get_shape().as_list() - else: - triang_shape = tf.shape(triang) - # The TT-rank could have changed: if qr_shape is e.g. 4 x 10, than q would - # be of size 4 x 4 and r would be 4 x 10, which means that the next rank - # should be changed to 4. - prev_rank = triang_shape[1] - if tt.is_tt_matrix(): - new_core_shape = (prev_rank, curr_mode_left, curr_mode_right, curr_rank) - else: - new_core_shape = (prev_rank, curr_mode, curr_rank) - tt_cores[core_idx] = tf.reshape(curr_core, new_core_shape) - - prev_core = tf.reshape(tt_cores[core_idx - 1], (-1, triang_shape[0])) - tt_cores[core_idx - 1] = tf.matmul(prev_core, triang) - - if tt.is_tt_matrix(): - first_core_shape = (1, raw_shape[0][0], raw_shape[1][0], prev_rank) - else: - first_core_shape = (1, raw_shape[0][0], prev_rank) - tt_cores[0] = tf.reshape(tt_cores[0], first_core_shape) - # TODO: infer the tt_ranks. - return TensorTrain(tt_cores, tt.get_raw_shape()) diff --git a/build/lib/t3f/decompositions_test.py b/build/lib/t3f/decompositions_test.py deleted file mode 100644 index 158eff9e..00000000 --- a/build/lib/t3f/decompositions_test.py +++ /dev/null @@ -1,176 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f import ops -from t3f import shapes -from t3f import decompositions -from t3f import initializers - - -class DecompositionsTest(tf.test.TestCase): - - def testTTTensor(self): - shape = (2, 1, 4, 3) - np.random.seed(1) - tens = np.random.rand(*shape).astype(np.float32) - tf_tens = tf.constant(tens) - tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=3) - with self.test_session(): - self.assertAllClose(tens, ops.full(tt_tens).eval()) - dynamic_tt_ranks = shapes.tt_ranks(tt_tens).eval() - static_tt_ranks = tt_tens.get_tt_ranks().as_list() - self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) - - # Try to decompose the same tensor with unknown shape. - tf_tens_pl = tf.placeholder(tf.float32, (None, None, 4, None)) - tt_tens = decompositions.to_tt_tensor(tf_tens_pl, max_tt_rank=3) - tt_val = ops.full(tt_tens).eval({tf_tens_pl: tens}) - self.assertAllClose(tens, tt_val) - dynamic_tt_ranks = shapes.tt_ranks(tt_tens).eval({tf_tens_pl: tens}) - self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) - - def testTTTensorSimple(self): - # Test that a tensor of ones and of zeros can be converted into TT with - # TT-rank 1. - shape = (2, 1, 4, 3) - tens_arr = (np.zeros(shape).astype(np.float32), - np.ones(shape).astype(np.float32)) - for tens in tens_arr: - tf_tens = tf.constant(tens) - tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=1) - with self.test_session(): - self.assertAllClose(tens, ops.full(tt_tens).eval()) - dynamic_tt_ranks = shapes.tt_ranks(tt_tens).eval() - static_tt_ranks = tt_tens.get_tt_ranks().as_list() - self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) - - # Try to decompose the same tensor with unknown shape. - tf_tens_pl = tf.placeholder(tf.float32, (None, None, None, None)) - tt_tens = decompositions.to_tt_tensor(tf_tens_pl, max_tt_rank=1) - tt_val = ops.full(tt_tens).eval({tf_tens_pl: tens}) - self.assertAllClose(tens, tt_val) - dynamic_tt_ranks = shapes.tt_ranks(tt_tens).eval({tf_tens_pl: tens}) - self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) - - def testTTVector(self): - vec_shape = (2, 1, 4, 3) - np.random.seed(1) - rows = np.prod(vec_shape) - vec = np.random.rand(rows, 1).astype(np.float32) - tf_vec = tf.constant(vec) - tt_vec = decompositions.to_tt_matrix(tf_vec, (vec_shape, None)) - with self.test_session(): - self.assertAllClose(vec, ops.full(tt_vec).eval()) - - def testTTMatrix(self): - # Convert a np.prod(out_shape) x np.prod(in_shape) matrix into TT-matrix - # and back. - inp_shape = (2, 5, 2, 3) - out_shape = (3, 3, 2, 3) - np.random.seed(1) - mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) - mat = mat.astype(np.float32) - tf_mat = tf.constant(mat) - tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), - max_tt_rank=90) - with self.test_session(): - # TODO: why so bad accuracy? - self.assertAllClose(mat, ops.full(tt_mat).eval(), atol=1e-5, rtol=1e-5) - - def testRoundTensor(self): - shape = (2, 1, 4, 3, 3) - np.random.seed(1) - tens = initializers.random_tensor(shape, tt_rank=15) - rounded_tens = decompositions.round(tens, max_tt_rank=9) - with self.test_session() as sess: - vars = [ops.full(tens), ops.full(rounded_tens)] - tens_value, rounded_tens_value = sess.run(vars) - # TODO: why so bad accuracy? - self.assertAllClose(tens_value, rounded_tens_value, atol=1e-4, rtol=1e-4) - dynamic_tt_ranks = shapes.tt_ranks(rounded_tens).eval() - self.assertAllEqual([1, 2, 2, 8, 3, 1], dynamic_tt_ranks) - - def testOrthogonalizeLeftToRight(self): - shape = (2, 4, 3, 3) - tt_ranks = (1, 5, 2, 17, 1) - updated_tt_ranks = (1, 2, 2, 6, 1) - tens = initializers.random_tensor(shape, tt_rank=tt_ranks) - orthogonal = decompositions.orthogonalize_tt_cores(tens) - with self.test_session() as sess: - tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) - self.assertAllClose(tens_val, orthogonal_val, atol=1e-5, rtol=1e-5) - dynamic_tt_ranks = shapes.tt_ranks(orthogonal).eval() - self.assertAllEqual(updated_tt_ranks, dynamic_tt_ranks) - # Check that the TT-cores are orthogonal. - for core_idx in range(4 - 1): - core = orthogonal.tt_cores[core_idx] - core = tf.reshape(core, (updated_tt_ranks[core_idx] * shape[core_idx], - updated_tt_ranks[core_idx + 1])) - should_be_eye = tf.matmul(tf.transpose(core), core) - should_be_eye_val = sess.run(should_be_eye) - self.assertAllClose(np.eye(updated_tt_ranks[core_idx + 1]), - should_be_eye_val) - - def testOrthogonalizeRightToLeft(self): - shape = (2, 4, 3, 3) - tt_ranks = (1, 5, 2, 17, 1) - updated_tt_ranks = (1, 5, 2, 3, 1) - tens = initializers.random_tensor(shape, tt_rank=tt_ranks) - orthogonal = decompositions.orthogonalize_tt_cores(tens, left_to_right=False) - with self.test_session() as sess: - tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) - self.assertAllClose(tens_val, orthogonal_val, atol=1e-5, rtol=1e-5) - dynamic_tt_ranks = shapes.tt_ranks(orthogonal).eval() - self.assertAllEqual(updated_tt_ranks, dynamic_tt_ranks) - # Check that the TT-cores are orthogonal. - for core_idx in range(1, 4): - core = orthogonal.tt_cores[core_idx] - core = tf.reshape(core, (updated_tt_ranks[core_idx], shape[core_idx] * - updated_tt_ranks[core_idx + 1])) - should_be_eye = tf.matmul(core, tf.transpose(core)) - should_be_eye_val = sess.run(should_be_eye) - self.assertAllClose(np.eye(updated_tt_ranks[core_idx]), - should_be_eye_val) - - -class DecompositionsBatchTest(tf.test.TestCase): - - def testOrthogonalizeLeftToRight(self): - shape = (2, 4, 3, 3) - tt_ranks = (1, 5, 2, 17, 1) - updated_tt_ranks = (1, 2, 2, 6, 1) - tens = initializers.random_tensor_batch(shape, tt_rank=tt_ranks, - batch_size=2) - orthogonal = decompositions.orthogonalize_tt_cores(tens) - with self.test_session() as sess: - tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) - self.assertAllClose(tens_val, orthogonal_val, atol=1e-5, rtol=1e-5) - dynamic_tt_ranks = shapes.tt_ranks(orthogonal).eval() - self.assertAllEqual(updated_tt_ranks, dynamic_tt_ranks) - # Check that the TT-cores are orthogonal. - for core_idx in range(4 - 1): - core_shape = (updated_tt_ranks[core_idx] * shape[core_idx], - updated_tt_ranks[core_idx + 1]) - for i in range(2): - core = tf.reshape(orthogonal.tt_cores[core_idx][i], core_shape) - should_be_eye = tf.matmul(tf.transpose(core), core) - should_be_eye_val = sess.run(should_be_eye) - self.assertAllClose(np.eye(updated_tt_ranks[core_idx + 1]), - should_be_eye_val) - - def testRoundTensor(self): - shape = (2, 1, 4, 3, 3) - tens = initializers.random_tensor_batch(shape, tt_rank=15, batch_size=3) - rounded_tens = decompositions.round(tens, max_tt_rank=9) - with self.test_session() as sess: - vars = [ops.full(tens), ops.full(rounded_tens)] - tens_value, rounded_tens_value = sess.run(vars) - # TODO: why so bad accuracy? - self.assertAllClose(tens_value, rounded_tens_value, atol=1e-4, - rtol=1e-4) - dynamic_tt_ranks = shapes.tt_ranks(rounded_tens).eval() - self.assertAllEqual([1, 2, 2, 8, 3, 1], dynamic_tt_ranks) - - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/examples_tests.py b/build/lib/t3f/examples_tests.py deleted file mode 100644 index 63dbc151..00000000 --- a/build/lib/t3f/examples_tests.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Tests from the README examples and the paper.""" - -import tensorflow as tf -import t3f - -class ExamplesTest(tf.test.TestCase): - - def testMainReadme(self): - # Just check that the readme examples do not raise exceptions. - # Create a random tensor of shape (3, 2, 2). - a = t3f.random_tensor((3, 2, 2), tt_rank=3) - norm = t3f.frobenius_norm(a) - # Convert TT-tensor into a dense tensor for printing. - a_full = t3f.full(a) - # Run a tensorflow session to run the operations. - with tf.Session() as sess: - # Run the operations. Note that if you run these - # two operations separetly (sess.run(a_full), sess.run(norm)) - # the result will be different, since sess.run will - # generate a new random tensor a on each run because `a' is - # an operation 'generate me a random tensor'. - a_val, norm_val = sess.run([a_full, norm]) - a = t3f.random_tensor((3, 2, 2), tt_rank=3) - b_dense = tf.random_normal((3, 2, 2)) - # Use TT-SVD on b_dense. - b_tt = t3f.to_tt_tensor(b_dense, max_tt_rank=4) - sum_round = t3f.round(t3f.add(a, b_tt), max_tt_rank=2) - # Inner product (sum of products of all elements). - a = t3f.random_tensor((3, 2, 2), tt_rank=3) - b = t3f.random_tensor((3, 2, 2), tt_rank=4) - inner_prod = t3f.flat_inner(a, b) - A = t3f.random_matrix(((3, 2, 2), (2, 3, 3)), tt_rank=3) - b = t3f.random_matrix(((2, 3, 3), None), tt_rank=3) - # Matrix-by-vector - matvec = t3f.matmul(A, b) - - # Matrix-by-dense matrix - b_dense = tf.random_normal((18, 1)) - matvec2 = t3f.matmul(A, b_dense) - - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/initializers.py b/build/lib/t3f/initializers.py deleted file mode 100644 index 6f192cc0..00000000 --- a/build/lib/t3f/initializers.py +++ /dev/null @@ -1,789 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f.tensor_train import TensorTrain -from t3f.tensor_train_batch import TensorTrainBatch -from t3f.tensor_train_base import TensorTrainBase -from t3f import shapes - - -def _validate_input_parameters(is_tensor, shape, **params): - """Internal function for validating input parameters - - Args: - is_tensor: bool, determines whether we attempt to construct a TT-tensor or - a TT-matrix (needed for the correct shape checks). - shape: array, the desired shape of the generated TT object - params: optional, possible values: - batch_size: int, for constructing batches - tt_rank: array or int, desired TT-ranks - """ - - if is_tensor: - if len(shape.shape) != 1: - raise ValueError('shape should be 1d array, got %a' % shape) - if np.any(shape < 1): - raise ValueError('all elements in `shape` should be positive, got %a' % - shape) - if not all(isinstance(sh, np.integer) for sh in shape): - raise ValueError('all elements in `shape` should be integers, got %a' % - shape) - else: - if len(shape.shape) != 2: - raise ValueError('shape should be 2d array, got %a' % shape) - if shape[0].size != shape[1].size: - raise ValueError('shape[0] should have the same length as shape[1], but' - 'got %d and %d' % (shape[0].size, shape[1].size)) - if np.any(shape.flatten() < 1): - raise ValueError('all elements in `shape` should be positive, got %a' % - shape) - if not all(isinstance(sh, np.integer) for sh in shape.flatten()): - raise ValueError('all elements in `shape` should be integers, got %a' % - shape) - - if 'batch_size' in params: - batch_size = params['batch_size'] - if not isinstance(batch_size, (int, np.integer)): - raise ValueError('`batch_size` should be integer, got %f' % batch_size) - if batch_size < 1: - raise ValueError('Batch size should be positive, got %d' % batch_size) - if 'tt_rank' in params: - tt_rank = params['tt_rank'] - if tt_rank.size == 1: - if not isinstance(tt_rank[()], np.integer): - raise ValueError('`tt_rank` should be integer, got %f' % tt_rank[()]) - if tt_rank.size > 1: - if not all(isinstance(tt_r, np.integer) for tt_r in tt_rank): - raise ValueError('all elements in `tt_rank` should be integers, got' - ' %a' % tt_rank) - if np.any(tt_rank < 1): - raise ValueError('`tt_rank` should be positive, got %a' % tt_rank) - - if is_tensor: - if tt_rank.size != 1 and tt_rank.size != (shape.size + 1): - raise ValueError('`tt_rank` array has inappropriate size, expected' - '1 or %d, got %d' % (shape.size + 1, tt_rank.size)) - else: - if tt_rank.size != 1 and tt_rank.size != (shape[0].size + 1): - raise ValueError('`tt_rank` array has inappropriate size, expected' - '1 or %d, got %d' % (shape[0].size + 1, tt_rank.size)) - - -def tensor_ones(shape): - """Generate TT-tensor of the given shape with all entries equal to 1. - - Args: - shape: array representing the shape of the future tensor - - Returns: - TensorTrain object containing a TT-tensor - """ - - shape = np.array(shape) - _validate_input_parameters(is_tensor=True, shape=shape) - num_dims = shape.size - tt_rank = np.ones(num_dims + 1) - - tt_cores = num_dims * [None] - for i in range(num_dims): - curr_core_shape = (1, shape[i], 1) - tt_cores[i] = tf.ones(curr_core_shape) - - return TensorTrain(tt_cores, shape, tt_rank) - - -def tensor_zeros(shape): - """Generate TT-tensor of the given shape with all entries equal to 0. - - Args: - shape: array representing the shape of the future tensor - - Returns: - TensorTrain object containing a TT-tensor - """ - - shape = np.array(shape) - _validate_input_parameters(is_tensor=True, shape=shape) - num_dims = shape.size - tt_rank = np.ones(num_dims + 1) - tt_cores = num_dims * [None] - for i in range(num_dims): - curr_core_shape = (1, shape[i], 1) - tt_cores[i] = tf.zeros(curr_core_shape) - - return TensorTrain(tt_cores, shape, tt_rank) - - -def eye(shape): - """Creates an identity TT-matrix. - - Args: - shape: array which defines the shape of the matrix row and column - indices. - - Returns: - TensorTrain containing an identity TT-matrix of size - np.prod(shape) x np.prod(shape) - """ - shape = np.array(shape) - # In this special case shape is in the same format as in the TT-tensor case - _validate_input_parameters(is_tensor=True, shape=shape) - - num_dims = shape.size - tt_ranks = np.ones(num_dims + 1) - - tt_cores = num_dims * [None] - for i in range(num_dims): - curr_core_shape = (1, shape[i], shape[i], 1) - tt_cores[i] = tf.reshape(tf.eye(shape[i]), curr_core_shape) - - true_shape = np.vstack([shape, shape]) - return TensorTrain(tt_cores, true_shape, tt_ranks) - - -def matrix_ones(shape): - """Generate a TT-matrix of the given shape with each entry equal to 1. - - Args: - shape: 2d array, shape[0] is the shape of the matrix row-index, - shape[1] is the shape of the column index. - shape[0] and shape[1] should have the same number of elements (d) - Also supports omitting one of the dimensions for vectors, e.g. - matrix_ones([[2, 2, 2], None]) - and - matrix_ones([None, [2, 2, 2]]) - will create an 8-element column and row vectors correspondingly. - - Returns: - TensorTrain containing a TT-matrix of size - np.prod(shape[0]) x np.prod(shape[1]) with each entry equal to 1 - """ - - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1]), dtype=int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0]), dtype=int) - shape = np.array(shape) - - _validate_input_parameters(is_tensor=False, shape=shape) - - num_dims = shape[0].size - tt_rank = np.ones(shape[0].size + 1) - - # TODO: variable (name?) scope. - - tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (1, shape[0][i], shape[1][i], 1) - tt_cores[i] = tf.ones(curr_core_shape) - - return TensorTrain(tt_cores, shape, tt_rank) - - -def matrix_zeros(shape): - """Generate a TT-matrix of the given shape with each entry equal to 0. - - Args: - shape: 2d array, shape[0] is the shape of the matrix row-index, - shape[1] is the shape of the column index. - shape[0] and shape[1] should have the same number of elements (d) - Also supports omitting one of the dimensions for vectors, e.g. - matrix_zeros([[2, 2, 2], None]) - and - matrix_zeros([None, [2, 2, 2]]) - will create an 8-element column and row vectors correspondingly. - - Returns: - TensorTrain containing a TT-matrix of size - np.prod(shape[0]) x np.prod(shape[1]) with each entry equal to 0 - """ - - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1]), dtype=int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0]), dtype=int) - shape = np.array(shape) - - _validate_input_parameters(is_tensor=False, shape=shape) - num_dims = shape[0].size - tt_rank = np.ones(shape[0].size + 1) - - # TODO: variable (name?) scope. - - tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (1, shape[0][i], shape[1][i], 1) - tt_cores[i] = tf.zeros(curr_core_shape) - - return TensorTrain(tt_cores, shape, tt_rank) - - -def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): - """Generate a TT-tensor of the given shape with N(mean, stddev^2) cores. - - Args: - shape: array representing the shape of the future tensor. - tt_rank: a number or a (d+1)-element array with the desired ranks. - mean: a number, the mean of the normal distribution used for - initializing TT-cores. - stddev: a number, the standard deviation of the normal distribution used - for initializing TT-cores. - - Returns: - TensorTrain containing a TT-tensor - """ - # TODO: good distribution to init training. - # TODO: support shape and tt_ranks as TensorShape?. - # TODO: support None as a dimension. - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=True, shape=shape, tt_rank=tt_rank) - num_dims = shape.size - if tt_rank.size == 1: - tt_rank = tt_rank * np.ones(num_dims - 1) - tt_rank = np.insert(tt_rank, 0, 1) - tt_rank = np.append(tt_rank, 1) - - tt_rank = tt_rank.astype(int) - # TODO: variable (name?) scope. - tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (tt_rank[i], shape[i], tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) - - return TensorTrain(tt_cores, shape, tt_rank) - - -def tensor_batch_with_random_cores(shape, tt_rank=2, batch_size=1, - mean=0., stddev=1.): - """Generate a batch of TT-tensors of given shape with N(mean, stddev^2) cores. - - Args: - shape: array representing the shape of the future tensor. - tt_rank: a number or a (d+1)-element array with ranks. - batch_size: an integer. - mean: a number, the mean of the normal distribution used for - initializing TT-cores. - stddev: a number, the standard deviation of the normal distribution used - for initializing TT-cores. - - Returns: - TensorTrainBatch containing TT-tensors - """ - - # TODO: support shape and tt_ranks as TensorShape?. - # TODO: support None as a dimension. - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=True, shape=shape, tt_rank=tt_rank, - batch_size=batch_size) - num_dims = shape.size - if tt_rank.size == 1: - tt_rank = tt_rank * np.ones(num_dims - 1) - tt_rank = np.insert(tt_rank, 0, 1) - tt_rank = np.append(tt_rank, 1) - tt_rank = tt_rank.astype(int) - # TODO: variable (name?) scope. - tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (batch_size, tt_rank[i], shape[i], tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) - - return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) - - -def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): - """Generate a TT-matrix of given shape with N(mean, stddev^2) cores. - - Args: - shape: 2d array, shape[0] is the shape of the matrix row-index, - shape[1] is the shape of the column index. - shape[0] and shape[1] should have the same number of elements (d) - Also supports omitting one of the dimensions for vectors, e.g. - matrix_with_random_cores([[2, 2, 2], None]) - and - matrix_with_random_cores([None, [2, 2, 2]]) - will create an 8-element column and row vectors correspondingly. - tt_rank: a number or a (d+1)-element array with ranks. - mean: a number, the mean of the normal distribution used for - initializing TT-cores. - stddev: a number, the standard deviation of the normal distribution used - for initializing TT-cores. - - Returns: - TensorTrain containing a TT-matrix of size - np.prod(shape[0]) x np.prod(shape[1]) - """ - # TODO: good distribution to init training. - # In case the shape is immutable. - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1]), dtype=int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0]), dtype=int) - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) - - num_dims = shape[0].size - if tt_rank.size == 1: - tt_rank = tt_rank * np.ones(num_dims - 1) - tt_rank = np.concatenate([[1], tt_rank, [1]]) - - tt_rank = tt_rank.astype(int) - # TODO: variable (name?) scope. - tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (tt_rank[i], shape[0][i], shape[1][i], - tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) - - return TensorTrain(tt_cores, shape, tt_rank) - - -def matrix_batch_with_random_cores(shape, tt_rank=2, batch_size=1, - mean=0., stddev=1.): - """Generate a batch of TT-matrices of given shape with N(mean, stddev^2) cores. - - Args: - shape: 2d array, shape[0] is the shape of the matrix row-index, - shape[1] is the shape of the column index. - shape[0] and shape[1] should have the same number of elements (d) - Also supports omitting one of the dimensions for vectors, e.g. - matrix_batch_with_random_cores([[2, 2, 2], None]) - and - matrix_batch_with_random_cores([None, [2, 2, 2]]) - will create a batch of one 8-element column and row vector correspondingly. - - tt_rank: a number or a (d+1)-element array with ranks. - batch_size: an integer. - mean: a number, the mean of the normal distribution used for - initializing TT-cores. - stddev: a number, the standard deviation of the normal distribution used - for initializing TT-cores. - - Returns: - TensorTrainBatch containing a batch of TT-matrices of size - np.prod(shape[0]) x np.prod(shape[1]) - """ - # TODO: good distribution to init training. - # In case the shape is immutable. - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1]), dtype=int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0]), dtype=int) - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank, - batch_size=batch_size) - num_dims = shape[0].size - if tt_rank.size == 1: - tt_rank = tt_rank * np.ones(num_dims - 1) - tt_rank = np.concatenate([[1], tt_rank, [1]]) - shape = shape.astype(int) - tt_rank = tt_rank.astype(int) - # TODO: variable (name?) scope. - tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (batch_size, tt_rank[i], shape[0][i], shape[1][i], - tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) - - return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) - - -def ones_like(tt): - """Constructs t3f.ones with the shape of `tt`. - - In the case when `tt` is TensorTrainBatch constructs t3f.ones with the shape - of a TensorTrain in `tt`. - - Args: - tt: TensorTrain object - - Returns: - TensorTrain object of the same shape as `tt` but with all entries equal to - 1. - - """ - if not isinstance(tt, TensorTrainBase): - raise ValueError("`tt` has to be a Tensor Train object") - else: - shape = shapes.lazy_raw_shape(tt) - if tt.is_tt_matrix(): - return matrix_ones(shape) - else: - return tensor_ones(shape[0, :]) - - -def zeros_like(tt): - """Constructs t3f.zeros with the shape of `tt`. - - In the case when `tt` is a TensorTrainBatch constructs t3f.zeros with - the shape of a TensorTrain in `tt`. - - Args: - tt: TensorTrain object - - Returns: - TensorTrain object of the same shape as `tt` but with all entries equal to - 0. - - """ - if not isinstance(tt, TensorTrainBase): - raise ValueError("`tt` has to be a Tensor Train object") - else: - shape = shapes.lazy_raw_shape(tt) - if tt.is_tt_matrix(): - return matrix_zeros(shape) - else: - return tensor_zeros(shape[0, :]) - - -def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): - """Generate a random TT-tensor of the given shape with given mean and stddev. - - Entries of the generated tensor (in the full format) will be iid and satisfy - E[x_{i1i2..id}] = mean, Var[x_{i1i2..id}] = stddev^2, but the distribution is - in fact not Gaussian (but is close for large tensors). - - In the current implementation only mean 0 is supported. To get - a random_tensor with specified mean but tt_rank greater by 1 you can - call - x = t3f.random_tensor(shape, tt_rank, stddev=stddev) - x = mean * t3f.ones_like(x) + x - - Args: - shape: array representing the shape of the future tensor. - tt_rank: a number or a (d+1)-element array with the desired ranks. - mean: a number, the desired mean for the distribution of entries. - stddev: a number, the desired standard deviation for the distribution of - entries. - - Returns: - TensorTrain containing a TT-tensor - """ - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=True, shape=shape, tt_rank=tt_rank) - - num_dims = shape.size - if tt_rank.size == 1: - tt_rank = tt_rank * np.ones(num_dims - 1) - tt_rank = np.insert(tt_rank, 0, 1) - tt_rank = np.append(tt_rank, 1) - - tt_rank = tt_rank.astype(int) - - # Empirically entries of a TT tensor with cores initialized from N(0, 1) - # will have variances np.prod(tt_rank) and mean 0. - # We scale each TT-core to obtain the desired stddev - - cr_exponent = -1.0 / (2 * num_dims) - var = np.prod(tt_rank ** cr_exponent) - core_stddev = stddev ** (1.0 / num_dims) * var - tt = tensor_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) - - if np.abs(mean) < 1e-8: - return tt - else: - raise NotImplementedError('non-zero mean is not supported yet') - - -def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): - """Generate a batch of TT-tensors with given shape, mean and stddev. - - Entries of the generated tensors (in the full format) will be iid and satisfy - E[x_{i1i2..id}] = mean, Var[x_{i1i2..id}] = stddev^2, but the distribution is - in fact not Gaussian (but is close for large tensors). - - In the current implementation only mean 0 is supported. To get - a random_tensor_batch with specified mean but tt_rank greater by 1 you can - call - x = t3f.random_tensor_batch(shape, tt_rank, batch_size=bs, stddev=stddev) - x = mean * t3f.ones_like(x) + x - - Args: - shape: array representing the shape of the future tensor. - tt_rank: a number or a (d+1)-element array with ranks. - batch_size: an integer. - mean: a number, the desired mean for the distribution of entries. - stddev: a number, the desired standard deviation for the distribution of - entries. - - Returns: - TensorTrainBatch containing TT-tensors. - """ - # TODO: support shape and tt_ranks as TensorShape?. - # TODO: support None as a dimension. - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=True, shape=shape, tt_rank=tt_rank, - batch_size=batch_size) - num_dims = shape.size - if tt_rank.size == 1: - tt_rank = tt_rank * np.ones(num_dims - 1) - tt_rank = np.insert(tt_rank, 0, 1) - tt_rank = np.append(tt_rank, 1) - tt_rank = tt_rank.astype(int) - - cr_exponent = -1.0 / (2 * num_dims) - var = np.prod(tt_rank ** cr_exponent) - cr_stddev = stddev ** (1.0 / num_dims) * var - tt = tensor_batch_with_random_cores(shape, tt_rank=tt_rank, stddev=cr_stddev, - batch_size=batch_size) - - if np.abs(mean) < 1e-8: - return tt - else: - raise NotImplementedError('non-zero mean is not supported yet') - - -def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): - """Generate a random TT-matrix of the given shape with given mean and stddev. - - Entries of the generated matrix (in the full format) will be iid and satisfy - E[x_{i1i2..id}] = mean, Var[x_{i1i2..id}] = stddev^2, but the distribution is - in fact not Gaussian. - - In the current implementation only mean 0 is supported. To get - a random_matrix with specified mean but tt_rank greater by 1 you can call - x = t3f.random_matrix(shape, tt_rank, stddev=stddev) - x = mean * t3f.ones_like(x) + x - - Args: - shape: 2d array, shape[0] is the shape of the matrix row-index, - shape[1] is the shape of the column index. - shape[0] and shape[1] should have the same number of elements (d) - Also supports omitting one of the dimensions for vectors, e.g. - random_matrix([[2, 2, 2], None]) - and - random_matrix([None, [2, 2, 2]]) - will create an 8-element column and row vectors correspondingly. - tt_rank: a number or a (d+1)-element array with ranks. - mean: a number, the desired mean for the distribution of entries. - stddev: a number, the desired standard deviation for the distribution of - entries. - - Returns: - TensorTrain containing a TT-matrix of size - np.prod(shape[0]) x np.prod(shape[1]) - """ - # TODO: good distribution to init training. - # In case the shape is immutable. - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1]), dtype=int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0]), dtype=int) - shape = np.array(shape) - tt_rank = np.array(tt_rank) - - _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) - - num_dims = shape[0].size - if tt_rank.size == 1: - tt_rank = tt_rank * np.ones(num_dims - 1) - tt_rank = np.concatenate([[1], tt_rank, [1]]) - - tt_rank = tt_rank.astype(int) - var = np.prod(tt_rank) - - # Empirically entries of a TT tensor with cores initialized from N(0, 1) - # will have variances np.prod(tt_rank) and mean 0. - # We scale each TT-core to obtain the desired stddev - - cr_exponent = -1.0 / (2 * num_dims) - var = np.prod(tt_rank ** cr_exponent) - core_stddev = stddev ** (1.0 / num_dims) * var - tt = matrix_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) - - if np.abs(mean) < 1e-8: - return tt - else: - raise NotImplementedError('non-zero mean is not supported yet') - - -def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): - """Generate a batch of TT-matrices with given shape, mean and stddev. - - Entries of the generated matrices (in the full format) will be iid and - satisfy E[x_{i1i2..id}] = mean, Var[x_{i1i2..id}] = stddev^2, but the - distribution is in fact not Gaussian. - - In the current implementation only mean 0 is supported. To get a - random_matrix_batch with specified mean but tt_rank greater by 1 you can call - x = t3f.random_matrix_batch(shape, tt_rank, batch_size=bs, stddev=stddev) - x = mean * t3f.ones_like(x) + x - - Args: - shape: 2d array, shape[0] is the shape of the matrix row-index, - shape[1] is the shape of the column index. - shape[0] and shape[1] should have the same number of elements (d) - Also supports omitting one of the dimensions for vectors, e.g. - random_matrix_batch([[2, 2, 2], None]) - and - random_matrix_batch([None, [2, 2, 2]]) - will create a batch of one 8-element column and row vector correspondingly. - tt_rank: a number or a (d+1)-element array with ranks. - batch_size: an integer. - mean: a number, the desired mean for the distribution of entries. - stddev: a number, the desired standard deviation for the distribution of - entries. - - Returns: - TensorTrainBatch containing a batch of TT-matrices of size - np.prod(shape[0]) x np.prod(shape[1]) - """ - # In case the shape is immutable. - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1]), dtype=int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0]), dtype=int) - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank, - batch_size=batch_size) - num_dims = shape[0].size - if tt_rank.size == 1: - tt_rank = tt_rank * np.ones(num_dims - 1) - tt_rank = np.concatenate([[1], tt_rank, [1]]) - - shape = shape.astype(int) - tt_rank = tt_rank.astype(int) - - cr_exponent = -1.0 / (2 * num_dims) - var = np.prod(tt_rank ** cr_exponent) - core_stddev = stddev ** (1.0 / num_dims) * var - tt = matrix_batch_with_random_cores(shape, tt_rank=tt_rank, - stddev=core_stddev, - batch_size=batch_size) - - if np.abs(mean) < 1e-8: - return tt - else: - raise NotImplementedError('non-zero mean is not supported yet') - - -def glorot_initializer(shape, tt_rank=2): - """Constructs a random TT matrix with entrywise variance 2.0 / (n_in + n_out) - - Args: - shape: 2d array, shape[0] is the shape of the matrix row-index, - shape[1] is the shape of the column index. - shape[0] and shape[1] should have the same number of elements (d) - Also supports omitting one of the dimensions for vectors, e.g. - glorot_initializer([[2, 2, 2], None]) - and - glorot_initializer([None, [2, 2, 2]]) - will create an 8-element column and row vectors correspondingly. - tt_rank: a number or a (d+1)-element array with ranks. - - Returns: - TensorTrain containing a TT-matrix of size - np.prod(shape[0]) x np.prod(shape[1]) - """ - - # In case the shape is immutable. - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1]), dtype=int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0]), dtype=int) - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) - n_in = np.prod(shape[0]) - n_out = np.prod(shape[1]) - lamb = 2.0 / (n_in + n_out) - - return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) - - -def he_initializer(shape, tt_rank=2): - """Constructs a random TT matrix with entrywise variance 2.0 / n_in - - Args: - shape: 2d array, shape[0] is the shape of the matrix row-index, - shape[1] is the shape of the column index. - shape[0] and shape[1] should have the same number of elements (d) - Also supports omitting one of the dimensions for vectors, e.g. - he_initializer([[2, 2, 2], None]) - and - he_initializer([None, [2, 2, 2]]) - will create an 8-element column and row vectors correspondingly. - tt_rank: a number or a (d+1)-element array with ranks. - - Returns: - TensorTrain containing a TT-matrix of size - np.prod(shape[0]) x np.prod(shape[1]) - """ - - # In case the shape is immutable. - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1]), dtype=int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0]), dtype=int) - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) - n_in = np.prod(shape[0]) - lamb = 2.0 / n_in - - return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) - - -def lecun_initializer(shape, tt_rank=2): - """Constructs a random TT matrix with entrywise variance 1.0 / n_in - - Args: - shape: 2d array, shape[0] is the shape of the matrix row-index, - shape[1] is the shape of the column index. - shape[0] and shape[1] should have the same number of elements (d) - Also supports omitting one of the dimensions for vectors, e.g. - lecun_initializer([[2, 2, 2], None]) - and - lecun_initializer([None, [2, 2, 2]]) - will create an 8-element column and row vectors correspondingly. - tt_rank: a number or a (d+1)-element array with ranks. - - Returns: - TensorTrain containing a TT-matrix of size - np.prod(shape[0]) x np.prod(shape[1]) - """ - - # In case the shape is immutable. - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1]), dtype=int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0]), dtype=int) - shape = np.array(shape) - tt_rank = np.array(tt_rank) - _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) - n_in = np.prod(shape[0]) - lamb = 1.0 / n_in - return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) diff --git a/build/lib/t3f/initializers_test.py b/build/lib/t3f/initializers_test.py deleted file mode 100644 index a555eeba..00000000 --- a/build/lib/t3f/initializers_test.py +++ /dev/null @@ -1,176 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f import initializers -from t3f import ops - - -class InitializersTest(tf.test.TestCase): - - def testTensorOnesAndZeros(self): - tt_ones = initializers.tensor_ones([2, 3, 4]) - tt_zeros = initializers.tensor_zeros([2, 3, 4]) - - ones_desired = np.ones((2, 3, 4)) - zeros_desired = np.zeros((2, 3, 4)) - with self.test_session() as sess: - tt_ones_full = sess.run(ops.full(tt_ones)) - tt_zeros_full = sess.run(ops.full(tt_zeros)) - self.assertAllClose(tt_ones_full, ones_desired) - self.assertAllClose(tt_zeros_full, zeros_desired) - bad_shapes = [[[2, 3]], [-1, 3], [0.1, 4]] - for shape in bad_shapes: - with self.assertRaises(ValueError): - initializers.tensor_ones(shape) - with self.assertRaises(ValueError): - initializers.tensor_zeros(shape) - - def testMatrixOnesAndZeros(self): - tt_ones = initializers.matrix_ones([[2, 3, 4], [1, 2, 5]]) - tt_zeros = initializers.matrix_zeros([[2, 3, 4], [1, 2, 5]]) - - ones_desired = np.ones((24, 10)) - zeros_desired = np.zeros((24, 10)) - - bad_shapes = [[[-1, 2, 3], [3, 4, 6]], [[1.5, 2, 4], [2, 5, 6]], - [[1], [2, 3]], [2, 3, 4]] - with self.test_session() as sess: - tt_ones_full = sess.run(ops.full(tt_ones)) - tt_zeros_full = sess.run(ops.full(tt_zeros)) - self.assertAllClose(tt_ones_full, ones_desired) - self.assertAllClose(tt_zeros_full, zeros_desired) - for shape in bad_shapes: - with self.assertRaises(ValueError): - initializers.matrix_ones(shape) - with self.assertRaises(ValueError): - initializers.matrix_zeros(shape) - - def testEye(self): - tt_eye = initializers.eye([4, 5, 6]) - eye_desired = np.eye(120) - with self.test_session() as sess: - eye_full = sess.run(ops.full(tt_eye)) - self.assertAllClose(eye_full, eye_desired) - bad_shapes = [[[2, 3]], [-1, 3], [0.1, 4]] - for shape in bad_shapes: - with self.assertRaises(ValueError): - initializers.eye(shape) - - def testOnesLikeAndZerosLike(self): - a = initializers.random_tensor([2, 3, 4]) - b = initializers.ones_like(a) - c = initializers.zeros_like(a) - var_list = [ops.full(b), ops.full(c)] - with self.test_session() as sess: - bf, cf = sess.run(var_list) - self.assertAllClose(bf, np.ones((2, 3, 4))) - self.assertAllClose(cf, np.zeros((2, 3, 4))) - with self.assertRaises(ValueError): - initializers.ones_like(1) - with self.assertRaises(ValueError): - initializers.zeros_like(1) - - def testRandomTensor(self): - shapes = [[3, 4], [3, 4], [3, 4], [3, 4], [1, -2], [1.1, 2], [[3, 4]]] - tt_ranks = [-2, 1.5, [2, 3, 4, 5], [1.5], 2, 2, 2] - bad_cases = zip(shapes, tt_ranks) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.random_tensor(case[0], tt_rank=case[1]) - - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.tensor_with_random_cores(case[0], tt_rank=case[1]) - - with self.assertRaises(NotImplementedError): - initializers.random_tensor([1, 2], mean=1.0) - - def testRandomMatrix(self): - shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], - [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]]] - tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3]] - bad_cases = zip(shapes, tt_ranks) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.random_matrix(case[0], tt_rank=case[1]) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.matrix_with_random_cores(case[0], tt_rank=case[1]) - with self.assertRaises(NotImplementedError): - initializers.random_matrix([[2, 3, 4], [1, 2, 3]], mean=1.0) - - def testRandomTensorBatch(self): - shapes = [[3, 4], [3, 4], [3, 4], [3, 4], [1, -2], [1.1, 2], [[3, 4]], - [1, 2], [3, 4]] - tt_ranks = [-2, 1.5, [2, 3, 4, 5], [1.5], 2, 2, 2, 2, 2] - bs = [1] * 7 + [-1] + [0.5] - bad_cases = zip(shapes, tt_ranks, bs) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.random_tensor_batch(case[0], tt_rank=case[1], - batch_size=case[2]) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.tensor_batch_with_random_cores(case[0], tt_rank=case[1], - batch_size=case[2]) - with self.assertRaises(NotImplementedError): - initializers.random_tensor_batch([1, 2, 3], mean=1.0) - - def testRandomMatrixBatch(self): - shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], - [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]]] - tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3], 2, 2] - bs = 7 * [1] + [-1] + [0.5] - bad_cases = zip(shapes, tt_ranks, bs) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.random_matrix_batch(case[0], tt_rank=case[1], - batch_size=case[2]) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.matrix_batch_with_random_cores(case[0], tt_rank=case[1], - batch_size=case[2]) - with self.assertRaises(NotImplementedError): - initializers.random_matrix_batch([[1, 2, 3], [1, 2, 3]], mean=1.0) - - def testGlorot(self): - shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], - [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]]] - tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3]] - bad_cases = zip(shapes, tt_ranks) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.glorot_initializer(case[0], tt_rank=case[1]) - - def testHe(self): - shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], - [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]]] - tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3]] - bad_cases = zip(shapes, tt_ranks) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.he_initializer(case[0], tt_rank=case[1]) - - def testLecun(self): - shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], - [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], - [[1, 2, 3], [1, 2, 3]]] - tt_ranks = [2, 2, 2, 2, -1, [[[1]]], [2.5, 3]] - bad_cases = zip(shapes, tt_ranks) - for case in bad_cases: - with self.assertRaises(ValueError): - initializers.lecun_initializer(case[0], tt_rank=case[1]) - - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/kronecker.py b/build/lib/t3f/kronecker.py deleted file mode 100644 index 0910c364..00000000 --- a/build/lib/t3f/kronecker.py +++ /dev/null @@ -1,237 +0,0 @@ -import tensorflow as tf - -from t3f.tensor_train import TensorTrain -from t3f.tensor_train_batch import TensorTrainBatch -from t3f import ops - - -def determinant(kron_a): - """Computes the determinant of a given Kronecker-factorized matrix. - - Note, that this method can suffer from overflow. - - Args: - kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a - batch of matrices of size N x N, factorized into a Kronecker product of - square matrices (all tt-ranks are 1 and all tt-cores are square). - - Returns: - A number or a Tensor with numbers for each element in the batch. - The determinant of the given matrix. - - Raises: - ValueError if the tt-cores of the provided matrix are not square, - or the tt-ranks are not 1. - """ - if not _is_kron(kron_a): - raise ValueError('The argument should be a Kronecker product (tt-ranks ' - 'should be 1)') - - shapes_defined = kron_a.get_shape().is_fully_defined() - if shapes_defined: - i_shapes = kron_a.get_raw_shape()[0] - j_shapes = kron_a.get_raw_shape()[1] - else: - i_shapes = ops.raw_shape(kron_a)[0] - j_shapes = ops.raw_shape(kron_a)[1] - - if shapes_defined: - if i_shapes != j_shapes: - raise ValueError('The argument should be a Kronecker product of square ' - 'matrices (tt-cores must be square)') - - is_batch = isinstance(kron_a, TensorTrainBatch) - pows = tf.cast(tf.reduce_prod(i_shapes), kron_a.dtype) - cores = kron_a.tt_cores - det = 1 - for core_idx in range(kron_a.ndims()): - core = cores[core_idx] - if is_batch: - core_det = tf.matrix_determinant(core[:, 0, :, :, 0]) - else: - core_det = tf.matrix_determinant(core[0, :, :, 0]) - core_pow = pows / i_shapes[core_idx].value - - det *= tf.pow(core_det, core_pow) - return det - - -def slog_determinant(kron_a): - """Computes the sign and log-det of a given Kronecker-factorized matrix. - - Args: - kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a - batch of matrices of size N x N, factorized into a Kronecker product of - square matrices (all tt-ranks are 1 and all tt-cores are square). - - Returns: - Two number or two Tensor with numbers for each element in the batch. - Sign of the determinant and the log-determinant of the given - matrix. If the determinant is zero, then sign will be 0 and logdet will be - -Inf. In all cases, the determinant is equal to sign * np.exp(logdet). - - Raises: - ValueError if the tt-cores of the provided matrix are not square, - or the tt-ranks are not 1. - """ - if not _is_kron(kron_a): - raise ValueError('The argument should be a Kronecker product ' - '(tt-ranks should be 1)') - - shapes_defined = kron_a.get_shape().is_fully_defined() - if shapes_defined: - i_shapes = kron_a.get_raw_shape()[0] - j_shapes = kron_a.get_raw_shape()[1] - else: - i_shapes = ops.raw_shape(kron_a)[0] - j_shapes = ops.raw_shape(kron_a)[1] - - if shapes_defined: - if i_shapes != j_shapes: - raise ValueError('The argument should be a Kronecker product of square ' - 'matrices (tt-cores must be square)') - - is_batch = isinstance(kron_a, TensorTrainBatch) - pows = tf.cast(tf.reduce_prod(i_shapes), kron_a.dtype) - logdet = 0. - det_sign = 1. - - for core_idx in range(kron_a.ndims()): - core = kron_a.tt_cores[core_idx] - if is_batch: - core_det = tf.matrix_determinant(core[:, 0, :, :, 0]) - else: - core_det = tf.matrix_determinant(core[0, :, :, 0]) - core_abs_det = tf.abs(core_det) - core_det_sign = tf.sign(core_det) - core_pow = pows / i_shapes[core_idx].value - logdet += tf.log(core_abs_det) * core_pow - det_sign *= core_det_sign**(core_pow) - return det_sign, logdet - - -def inv(kron_a): - """Computes the inverse of a given Kronecker-factorized matrix. - - Args: - kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a - batch of matrices of size N x N, factorized into a Kronecker product of - square matrices (all tt-ranks are 1 and all tt-cores are square). - - Returns: - `TensorTrain` object containing a TT-matrix of size N x N if the argument is - `TensorTrain` - `TensorTrainBatch` object, containing TT-matrices of size N x N if the - argument is `TensorTrainBatch` - - Raises: - ValueError if the tt-cores of the provided matrix are not square, - or the tt-ranks are not 1. - """ - if not _is_kron(kron_a): - raise ValueError('The argument should be a Kronecker product ' - '(tt-ranks should be 1)') - - shapes_defined = kron_a.get_shape().is_fully_defined() - if shapes_defined: - i_shapes = kron_a.get_raw_shape()[0] - j_shapes = kron_a.get_raw_shape()[1] - else: - i_shapes = ops.raw_shape(kron_a)[0] - j_shapes = ops.raw_shape(kron_a)[1] - - if shapes_defined: - if i_shapes != j_shapes: - raise ValueError('The argument should be a Kronecker product of square ' - 'matrices (tt-cores must be square)') - - is_batch = isinstance(kron_a, TensorTrainBatch) - inv_cores = [] - for core_idx in range(kron_a.ndims()): - core = kron_a.tt_cores[core_idx] - if is_batch: - core_inv = tf.matrix_inverse(core[:, 0, :, :, 0]) - core_inv = tf.expand_dims(tf.expand_dims(core_inv, 1), -1) - else: - core_inv = tf.matrix_inverse(core[0, :, :, 0]) - core_inv = tf.expand_dims(tf.expand_dims(core_inv, 0), -1) - inv_cores.append(core_inv) - - res_ranks = kron_a.get_tt_ranks() - res_shape = kron_a.get_raw_shape() - if is_batch: - return TensorTrainBatch(inv_cores, res_shape, res_ranks) - else: - return TensorTrain(inv_cores, res_shape, res_ranks) - - -def cholesky(kron_a): - """Computes the Cholesky decomposition of a given Kronecker-factorized matrix. - - Args: - kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a - batch of matrices of size N x N, factorized into a Kronecker product of - square matrices (all tt-ranks are 1 and all tt-cores are square). All the - cores must be symmetric positive-definite. - - Returns: - `TensorTrain` object containing a TT-matrix of size N x N if the argument is - `TensorTrain` - `TensorTrainBatch` object, containing TT-matrices of size N x N if the - argument is `TensorTrainBatch` - - Raises: - ValueError if the tt-cores of the provided matrix are not square, - or the tt-ranks are not 1. - """ - if not _is_kron(kron_a): - raise ValueError('The argument should be a Kronecker product ' - '(tt-ranks should be 1)') - - shapes_defined = kron_a.get_shape().is_fully_defined() - if shapes_defined: - i_shapes = kron_a.get_raw_shape()[0] - j_shapes = kron_a.get_raw_shape()[1] - else: - i_shapes = ops.raw_shape(kron_a)[0] - j_shapes = ops.raw_shape(kron_a)[1] - - if shapes_defined: - if i_shapes != j_shapes: - raise ValueError('The argument should be a Kronecker product of square ' - 'matrices (tt-cores must be square)') - - is_batch = isinstance(kron_a, TensorTrainBatch) - cho_cores = [] - - for core_idx in range(kron_a.ndims()): - core = kron_a.tt_cores[core_idx] - if is_batch: - core_cho = tf.cholesky(core[:, 0, :, :, 0]) - core_cho = tf.expand_dims(tf.expand_dims(core_cho, 1), -1) - else: - core_cho = tf.cholesky(core[0, :, :, 0]) - core_cho = tf.expand_dims(tf.expand_dims(core_cho, 0), -1) - cho_cores.append(core_cho) - - res_ranks = kron_a.get_tt_ranks() - res_shape = kron_a.get_raw_shape() - if is_batch: - return TensorTrainBatch(cho_cores, res_shape, res_ranks) - else: - return TensorTrain(cho_cores, res_shape, res_ranks) - - -def _is_kron(tt_a): - """Returns True if the argument is a Kronecker product matrix. - - Args: - t_a: `TensorTrain` or `TensorTrainBatch` object. - - Returns: - bool - """ - if tt_a.is_tt_matrix(): - return max(tt_a.get_tt_ranks()) == 1 - return False - diff --git a/build/lib/t3f/kronecker_test.py b/build/lib/t3f/kronecker_test.py deleted file mode 100644 index 84a07940..00000000 --- a/build/lib/t3f/kronecker_test.py +++ /dev/null @@ -1,182 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f.tensor_train import TensorTrain -from t3f.tensor_train_batch import TensorTrainBatch -from t3f import ops -from t3f import initializers -from t3f import variables -from t3f import kronecker as kr - -class KroneckerTest(tf.test.TestCase): - - def testIsKronNonKron(self): - # Tests _is_kron on a non-Kronecker matrix - initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=2) - tt_mat = variables.get_variable('tt_mat', initializer=initializer) - self.assertFalse(kr._is_kron(tt_mat)) - - def testIsKronKron(self): - # Tests _is_kron on a Kronecker matrix - initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=1) - kron_mat = variables.get_variable('kron_mat', initializer=initializer) - self.assertTrue(kr._is_kron(kron_mat)) - - def testDet(self): - # Tests the determinant function - initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1) - kron_mat = variables.get_variable('kron_mat', initializer=initializer) - init_op = tf.global_variables_initializer() - with self.test_session() as sess: - sess.run(init_op) - desired = np.linalg.det(ops.full(kron_mat).eval()) - actual = kr.determinant(kron_mat).eval() - self.assertAllClose(desired, actual) - - def testSlogDet(self): - # Tests the slog_determinant function - - # TODO: use kron and -1 * kron matrices, when mul is implemented - # the current version is platform-dependent - - tf.set_random_seed(5) # negative derminant - initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1) - kron_neg = variables.get_variable('kron_neg', initializer=initializer) - - tf.set_random_seed(1) # positive determinant - initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1) - kron_pos = variables.get_variable('kron_pos', initializer=initializer) - - init_op = tf.global_variables_initializer() - with self.test_session() as sess: - # negative derminant - sess.run(init_op) - desired_sign, desired_det = np.linalg.slogdet(ops.full(kron_neg).eval()) - actual_sign, actual_det = sess.run(kr.slog_determinant(kron_neg)) - self.assertEqual(desired_sign, actual_sign) - self.assertAllClose(desired_det, actual_det) - - # positive determinant - desired_sign, desired_det = np.linalg.slogdet(ops.full(kron_pos).eval()) - actual_sign, actual_det = sess.run(kr.slog_determinant(kron_pos)) - self.assertEqual(desired_sign, actual_sign) - self.assertAllClose(desired_det, actual_det) - - def testInv(self): - # Tests the inv function - initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1) - kron_mat = variables.get_variable('kron_mat', initializer=initializer) - init_op = tf.global_variables_initializer() - with self.test_session() as sess: - sess.run(init_op) - desired = np.linalg.inv(ops.full(kron_mat).eval()) - actual = ops.full(kr.inv(kron_mat)).eval() - self.assertAllClose(desired, actual) - - def testCholesky(self): - # Tests the cholesky function - np.random.seed(8) - - # generating two symmetric positive-definite tt-cores - L_1 = np.tril(np.random.normal(scale=2., size=(2, 2))) - L_2 = np.tril(np.random.normal(scale=2., size=(3, 3))) - K_1 = L_1.dot(L_1.T) - K_2 = L_2.dot(L_2.T) - K = np.kron(K_1, K_2) - initializer = TensorTrain([K_1[None, :, :, None], - K_2[None, :, :, None]], - tt_ranks=7*[1]) - kron_mat = variables.get_variable('kron_mat', initializer=initializer) - init_op = tf.global_variables_initializer() - with self.test_session() as sess: - sess.run(init_op) - desired = np.linalg.cholesky(K) - actual = ops.full(kr.cholesky(kron_mat)).eval() - self.assertAllClose(desired, actual) - -class BatchKroneckerTest(tf.test.TestCase): - - def testIsKronNonKron(self): - # Tests _is_kron on a non-Kronecker matrix batch - initializer = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=2, - batch_size=3) - tt_mat_batch = variables.get_variable('tt_mat_batch', - initializer=initializer) - self.assertFalse(kr._is_kron(tt_mat_batch)) - - def testIsKronKron(self): - # Tests _is_kron on a Kronecker matrix batch - initializer = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=1, - batch_size=3) - kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) - self.assertTrue(kr._is_kron(kron_mat_batch)) - - def testDet(self): - # Tests the determinant function - initializer = initializers.random_matrix_batch(((2, 3, 2), (2, 3, 2)), - tt_rank=1, batch_size=3) - kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) - init_op = tf.global_variables_initializer() - with self.test_session() as sess: - sess.run(init_op) - desired = tf.matrix_determinant(ops.full(kron_mat_batch)).eval() - actual = kr.determinant(kron_mat_batch).eval() - self.assertAllClose(desired, actual) - - def testSlogDet(self): - # Tests the slog_determinant function - - tf.set_random_seed(1) # negative and positive determinants - initializer = initializers.random_matrix_batch(((2, 3), (2, 3)), tt_rank=1, - batch_size=3) - kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) - - init_op = tf.global_variables_initializer() - with self.test_session() as sess: - # negative derminant - sess.run(init_op) - desired_sign, desired_det = np.linalg.slogdet( - ops.full(kron_mat_batch).eval()) - actual_sign, actual_det = sess.run(kr.slog_determinant(kron_mat_batch)) - self.assertAllEqual(desired_sign, actual_sign) - self.assertAllClose(desired_det, actual_det) - - def testInv(self): - # Tests the inv function - initializer = initializers.random_matrix_batch(((2, 3, 2), (2, 3, 2)), - tt_rank=1, batch_size=3) - kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) - init_op = tf.global_variables_initializer() - with self.test_session() as sess: - sess.run(init_op) - desired = np.linalg.inv(ops.full(kron_mat_batch).eval()) - actual = ops.full(kr.inv(kron_mat_batch)).eval() - self.assertAllClose(desired, actual, atol=1e-4) - - def testCholesky(self): - # Tests the cholesky function - np.random.seed(8) - - # generating two symmetric positive-definite tt-cores - L_1 = np.tril(np.random.normal(scale=2., size=(4, 2, 2))) - L_2 = np.tril(np.random.normal(scale=2., size=(4, 3, 3))) - K_1 = np.einsum('ijk,ilk->ijl', L_1, L_1) - K_2 = np.einsum('ijk,ilk->ijl', L_2, L_2) - initializer = TensorTrainBatch([K_1[:, None, :, :, None], - K_2[:, None, :, :, None]], tt_ranks=7*[1]) - kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) - init_op = tf.global_variables_initializer() - with self.test_session() as sess: - sess.run(init_op) - desired = np.linalg.cholesky(ops.full(kron_mat_batch).eval()) - actual = ops.full(kr.cholesky(kron_mat_batch)).eval() - self.assertAllClose(desired, actual) - - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/ops.py b/build/lib/t3f/ops.py deleted file mode 100644 index 0ea892db..00000000 --- a/build/lib/t3f/ops.py +++ /dev/null @@ -1,1204 +0,0 @@ -import tensorflow as tf -import numpy as np -from t3f.tensor_train_base import TensorTrainBase -from t3f.tensor_train import TensorTrain -from t3f.tensor_train_batch import TensorTrainBatch -from t3f import shapes -from t3f import utils -from t3f import decompositions -from t3f import initializers - -# TODO: add complexities to the comments. - - -def full(tt): - """Converts a TensorTrain into a regular tensor or matrix (tf.Tensor). - - Args: - tt: `TensorTrain` or `TensorTrainBatch` object. - - Returns: - tf.Tensor. - """ - if isinstance(tt, TensorTrainBatch): - # Batch of Tensor Trains. - return _full_tt_batch(tt) - else: - # TensorTrain object (not batch). - return _full_tt(tt) - - -def _full_tt(tt): - """Converts a TensorTrain into a regular tensor or matrix (tf.Tensor). - - Args: - tt: `TensorTrain` object. - - Returns: - tf.Tensor. - """ - num_dims = tt.ndims() - ranks = shapes.lazy_tt_ranks(tt) - shape = shapes.lazy_shape(tt) - raw_shape = shapes.lazy_raw_shape(tt) - - res = tt.tt_cores[0] - for i in range(1, num_dims): - res = tf.reshape(res, (-1, ranks[i])) - curr_core = tf.reshape(tt.tt_cores[i], (ranks[i], -1)) - res = tf.matmul(res, curr_core) - if tt.is_tt_matrix(): - intermediate_shape = [] - for i in range(num_dims): - intermediate_shape.append(raw_shape[0][i]) - intermediate_shape.append(raw_shape[1][i]) - res = tf.reshape(res, intermediate_shape) - transpose = [] - for i in range(0, 2 * num_dims, 2): - transpose.append(i) - for i in range(1, 2 * num_dims, 2): - transpose.append(i) - res = tf.transpose(res, transpose) - return tf.reshape(res, shape) - else: - return tf.reshape(res, shape) - - -def _full_tt_batch(tt): - """Converts a TensorTrainBatch into a regular tensor or matrix (tf.Tensor). - - Args: - tt: `TensorTrainBatch` object. - - Returns: - tf.Tensor. - """ - num_dims = tt.ndims() - ranks = shapes.lazy_tt_ranks(tt) - shape = shapes.lazy_shape(tt) - raw_shape = shapes.lazy_raw_shape(tt) - - res = tt.tt_cores[0] - batch_size = shapes.lazy_batch_size(tt) - for i in range(1, num_dims): - res = tf.reshape(res, (batch_size, -1, ranks[i])) - curr_core = tf.reshape(tt.tt_cores[i], (batch_size, ranks[i], -1)) - res = tf.einsum('oqb,obw->oqw', res, curr_core) - if tt.is_tt_matrix(): - intermediate_shape = [batch_size] - for i in range(num_dims): - intermediate_shape.append(raw_shape[0][i]) - intermediate_shape.append(raw_shape[1][i]) - res = tf.reshape(res, intermediate_shape) - transpose = [0] - for i in range(0, 2 * num_dims, 2): - transpose.append(i + 1) - for i in range(1, 2 * num_dims, 2): - transpose.append(i + 1) - res = tf.transpose(res, transpose) - return tf.reshape(res, shape) - else: - return tf.reshape(res, shape) - - -def tt_tt_matmul(tt_matrix_a, tt_matrix_b): - """Multiplies two TT-matrices and returns the TT-matrix of the result. - - Args: - tt_matrix_a: `TensorTrain` or `TensorTrainBatch` object containing - a TT-matrix (a batch of TT-matrices) of size M x N - tt_matrix_b: `TensorTrain` or `TensorTrainBatch` object containing - a TT-matrix (a batch of TT-matrices) of size N x P - - Returns - `TensorTrain` object containing a TT-matrix of size M x P if both arguments - are `TensorTrain`s - `TensorTrainBatch` if any of the arguments is a `TensorTrainBatch` - - Raises: - ValueError is the arguments are not TT matrices or if their sizes are not - appropriate for a matrix-by-matrix multiplication. - """ - # Both TensorTrain and TensorTrainBatch are inherited from TensorTrainBase. - if not isinstance(tt_matrix_a, TensorTrainBase) or \ - not isinstance(tt_matrix_b, TensorTrainBase) or \ - not tt_matrix_a.is_tt_matrix() or \ - not tt_matrix_b.is_tt_matrix(): - raise ValueError('Arguments should be TT-matrices') - - if not shapes.is_batch_broadcasting_possible(tt_matrix_a, tt_matrix_b): - raise ValueError('The batch sizes are different and not 1, broadcasting is ' - 'not available.') - - ndims = tt_matrix_a.ndims() - if tt_matrix_b.ndims() != ndims: - raise ValueError('Arguments should have the same number of dimensions, ' - 'got %d and %d instead.' % (ndims, tt_matrix_b.ndims())) - - # Convert BatchSize 1 batch into TT object to simplify broadcasting. - tt_matrix_a = shapes.squeeze_batch_dim(tt_matrix_a) - tt_matrix_b = shapes.squeeze_batch_dim(tt_matrix_b) - is_a_batch = isinstance(tt_matrix_a, TensorTrainBatch) - is_b_batch = isinstance(tt_matrix_b, TensorTrainBatch) - is_res_batch = is_a_batch or is_b_batch - a_batch_str = 'o' if is_a_batch else '' - b_batch_str = 'o' if is_b_batch else '' - res_batch_str = 'o' if is_res_batch else '' - einsum_str = '{}aijb,{}cjkd->{}acikbd'.format(a_batch_str, b_batch_str, - res_batch_str) - result_cores = [] - # TODO: name the operation and the resulting tensor. - a_shape = shapes.lazy_raw_shape(tt_matrix_a) - a_ranks = shapes.lazy_tt_ranks(tt_matrix_a) - b_shape = shapes.lazy_raw_shape(tt_matrix_b) - b_ranks = shapes.lazy_tt_ranks(tt_matrix_b) - if is_res_batch: - if is_a_batch: - batch_size = shapes.lazy_batch_size(tt_matrix_a) - if is_b_batch: - batch_size = shapes.lazy_batch_size(tt_matrix_b) - for core_idx in range(ndims): - a_core = tt_matrix_a.tt_cores[core_idx] - b_core = tt_matrix_b.tt_cores[core_idx] - curr_res_core = tf.einsum(einsum_str, a_core, b_core) - - res_left_rank = a_ranks[core_idx] * b_ranks[core_idx] - res_right_rank = a_ranks[core_idx + 1] * b_ranks[core_idx + 1] - left_mode = a_shape[0][core_idx] - right_mode = b_shape[1][core_idx] - if is_res_batch: - core_shape = (batch_size, res_left_rank, left_mode, right_mode, res_right_rank) - else: - core_shape = (res_left_rank, left_mode, right_mode, - res_right_rank) - curr_res_core = tf.reshape(curr_res_core, core_shape) - result_cores.append(curr_res_core) - - res_shape = (tt_matrix_a.get_raw_shape()[0], tt_matrix_b.get_raw_shape()[1]) - static_a_ranks = tt_matrix_a.get_tt_ranks() - static_b_ranks = tt_matrix_b.get_tt_ranks() - out_ranks = [a_r * b_r for a_r, b_r in zip(static_a_ranks, static_b_ranks)] - if is_res_batch: - return TensorTrainBatch(result_cores, res_shape, out_ranks, batch_size) - else: - return TensorTrain(result_cores, res_shape, out_ranks) - - -def tt_dense_matmul(tt_matrix_a, matrix_b): - """Multiplies a TT-matrix by a regular matrix, returns a regular matrix. - - Args: - tt_matrix_a: `TensorTrain` object containing a TT-matrix of size M x N - matrix_b: tf.Tensor of size N x P - - Returns - tf.Tensor of size M x P - """ - if not isinstance(tt_matrix_a, TensorTrain) or not tt_matrix_a.is_tt_matrix(): - raise ValueError('The first argument should be a TT-matrix') - - ndims = tt_matrix_a.ndims() - a_columns = tt_matrix_a.get_shape()[1].value - b_rows = matrix_b.get_shape()[0].value - if a_columns is not None and b_rows is not None: - if a_columns != b_rows: - raise ValueError('Arguments shapes should align got %d and %d instead.' % - (tt_matrix_a.get_shape(), matrix_b.get_shape())) - - a_shape = shapes.lazy_shape(tt_matrix_a) - a_raw_shape = shapes.lazy_raw_shape(tt_matrix_a) - if matrix_b.get_shape().is_fully_defined(): - b_shape = matrix_b.get_shape().as_list() - else: - b_shape = tf.shape(matrix_b) - a_ranks = shapes.lazy_tt_ranks(tt_matrix_a) - # If A is (i0, ..., id-1) x (j0, ..., jd-1) and B is (j0, ..., jd-1) x K, - # data is (K, j0, ..., jd-2) x jd-1 x 1 - data = tf.transpose(matrix_b) - data = tf.reshape(data, (-1, a_raw_shape[1][-1], 1)) - for core_idx in reversed(range(ndims)): - curr_core = tt_matrix_a.tt_cores[core_idx] - # On the k = core_idx iteration, after applying einsum the shape of data - # becomes ik x (ik-1..., id-1, K, j0, ..., jk-1) x rank_k - data = tf.einsum('aijb,rjb->ira', curr_core, data) - if core_idx > 0: - # After reshape the shape of data becomes - # (ik, ..., id-1, K, j0, ..., jk-2) x jk-1 x rank_k - new_data_shape = (-1, a_raw_shape[1][core_idx - 1], a_ranks[core_idx]) - data = tf.reshape(data, new_data_shape) - # At the end the shape of the data is (i0, ..., id-1) x K - return tf.reshape(data, (a_shape[0], b_shape[1])) - - -def dense_tt_matmul(matrix_a, tt_matrix_b): - """Multiplies a regular matrix by a TT-matrix, returns a regular matrix. - - Args: - matrix_a: tf.Tensor of size M x N - tt_matrix_b: `TensorTrain` object containing a TT-matrix of size N x P - - Returns - tf.Tensor of size M x P - """ -# TODO: make a more efficient implementation. - a_t = tf.transpose(matrix_a) - b_t = transpose(tt_matrix_b) - return tf.transpose(tt_dense_matmul(b_t, a_t)) - - -def sparse_tt_matmul(sparse_matrix_a, tt_matrix_b): - """Multiplies a sparse matrix by a TT-matrix, returns a regular matrix. - - Args: - sparse_matrix_a: tf.SparseTensor of size M x N - tt_matrix_b: `TensorTrain` object containing a TT-matrix of size N x P - - Returns - tf.Tensor of size M x P - """ - raise NotImplementedError - - -# TODO: add flag `return_type = (TT | dense)`? -def tt_sparse_matmul(tt_matrix_a, sparse_matrix_b): - """Multiplies a TT-matrix by a sparse matrix, returns a regular matrix. - - Args: - tt_matrix_a: `TensorTrain` object containing a TT-matrix of size M x N - sparse_matrix_b: tf.SparseTensor of size N x P - - Returns - tf.Tensor of size M x P - """ - raise NotImplementedError - - -def matmul(a, b): - """Multiplies two matrices that can be TT-, dense, or sparse. - - Note that multiplication of two TT-matrices returns a TT-matrix with much - larger ranks. - Also works for multiplying two batches of TT-matrices or a product between a - TT-matrix and a batch of TT-matrices (with broadcasting). - - Args: - a: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor of - size M x N - b: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor of - size N x P - - Returns - If both arguments are `TensorTrain` objects, returns a `TensorTrain` - object containing a TT-matrix of size M x P. - If at least one of the arguments is a `TensorTrainBatch` object, returns - a `TensorTrainBatch` object containing a batch of TT-matrices of size - M x P. - Otherwise, returns tf.Tensor of size M x P. - """ -# TODO: is it safe to check types? What if a class is derived from TT? - if isinstance(a, TensorTrainBase) and isinstance(b, TensorTrainBase): - return tt_tt_matmul(a, b) - elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): - return tt_dense_matmul(a, b) - elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): - return dense_tt_matmul(a, b) - elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): - return tt_sparse_matmul(a, b) - elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): - return sparse_tt_matmul(a, b) - else: - raise ValueError('Argument types are not supported in matmul: %s x %s' % - (a, b)) - - -def tt_tt_flat_inner(tt_a, tt_b): - """Inner product between two TT-tensors or TT-matrices along all axis. - - The shapes of tt_a and tt_b should coincide. - - Args: - tt_a: `TensorTrain` or `TensorTrainBatch` object - tt_b: `TensorTrain` or `TensorTrainBatch` object - - Returns - a number or a Tensor with numbers for each element in the batch. - sum of products of all the elements of tt_a and tt_b - - Raises: - ValueError if the arguments are not `TensorTrain` objects, have different - number of TT-cores, different underlying shape, or if you are trying to - compute inner product between a TT-matrix and a TT-tensor. - - Complexity: - Multiplying two single TT-objects is O(d r^3 n) where d is the number of - TT-cores (tt_a.ndims()), r is the largest TT-rank - max(tt_a.get_tt_rank(), tt_b.get_tt_rank()) - and n is the size of the axis dimension, e.g. - for a tensor of size 4 x 4 x 4, n is 4; - for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 - A more precise complexity is O(d r1 r2 n max(r1, r2)) where - r1 is the largest TT-rank of tt_a and r2 is the largest TT-rank of tt_b. - The complexity of this operation for batch input is O(batch_size d r^3 n). - """ - if not isinstance(tt_a, TensorTrainBase) or not isinstance(tt_b, - TensorTrainBase): - raise ValueError('Arguments should be TensorTrains') - - if tt_a.is_tt_matrix() != tt_b.is_tt_matrix(): - raise ValueError('One of the arguments is a TT-tensor, the other is ' - 'a TT-matrix, disallowed') - are_both_matrices = tt_a.is_tt_matrix() and tt_b.is_tt_matrix() - - if not shapes.is_batch_broadcasting_possible(tt_a, tt_b): - raise ValueError('The batch sizes are different and not 1, broadcasting is ' - 'not available.') - - # TODO: compare shapes and raise if not consistent. - - ndims = tt_a.ndims() - if tt_b.ndims() != ndims: - raise ValueError('Arguments should have the same number of dimensions, ' - 'got %d and %d instead.' % (ndims, tt_b.ndims())) - - axes_str = 'ij' if are_both_matrices else 'i' - # Convert BatchSize 1 batch into TT object to simplify broadcasting. - tt_a = shapes.squeeze_batch_dim(tt_a) - tt_b = shapes.squeeze_batch_dim(tt_b) - is_a_batch = isinstance(tt_a, TensorTrainBatch) - is_b_batch = isinstance(tt_b, TensorTrainBatch) - is_res_batch = is_a_batch or is_b_batch - a_batch_str = 'o' if is_a_batch else '' - b_batch_str = 'o' if is_b_batch else '' - res_batch_str = 'o' if is_res_batch else '' - init_einsum_str = '{1}a{0}b,{2}c{0}d->{3}bd'.format(axes_str, a_batch_str, - b_batch_str, - res_batch_str) - a_core = tt_a.tt_cores[0] - b_core = tt_b.tt_cores[0] - # Simplest example of this operation: - # if both arguments are TT-tensors, then it is - # res = tf.einsum('aib,cid->bd', a_core, b_core) - res = tf.einsum(init_einsum_str, a_core, b_core) - # TODO: name the operation and the resulting tensor. - - einsum_str = '{3}ac,{1}a{0}b,{2}c{0}d->{3}bd'.format(axes_str, a_batch_str, - b_batch_str, - res_batch_str) - for core_idx in range(1, ndims): - a_core = tt_a.tt_cores[core_idx] - b_core = tt_b.tt_cores[core_idx] - # Simplest example of this operation: - # if both arguments are TT-tensors, then it is - # res = tf.einsum('ac,aib,cid->bd', res, a_core, b_core) - res = tf.einsum(einsum_str, res, a_core, b_core) - return tf.squeeze(res) - - -def tt_dense_flat_inner(tt_a, dense_b): - """Inner product between a TT-tensor (or TT-matrix) and tf.Tensor along all axis. - - The shapes of tt_a and dense_b should coincide. - - Args: - tt_a: `TensorTrain` object - dense_b: tf.Tensor - - Returns - a number - sum of products of all the elements of tt_a and dense_b - """ - raise NotImplementedError - - -def tt_sparse_flat_inner(tt_a, sparse_b): - """Inner product between a TT-tensor (or TT-matrix) and tf.SparseTensor along all axis. - - The shapes of tt_a and sparse_b should coincide. - - Args: - tt_a: `TensorTrain` object - sparse_b: tf.SparseTensor - - Returns - a number - sum of products of all the elements of tt_a and sparse_b - """ - if sparse_b.indices.get_shape().is_fully_defined(): - num_elements = sparse_b.indices.get_shape()[0] - else: - num_elements = tf.shape(sparse_b.indices)[0] - a_shape = shapes.lazy_raw_shape(tt_a) - a_ranks = shapes.lazy_tt_ranks(tt_a) - if tt_a.is_tt_matrix(): - tt_a_elements = tf.ones((num_elements, 1, 1)) - # TODO: use t3f.shape is safer?? - tensor_shape = tt_a.get_raw_shape() - row_idx_linear = tf.cast(sparse_b.indices[:, 0], tf.int64) - row_idx = utils.unravel_index(row_idx_linear, tf.cast(tensor_shape[0], tf.int64)) - col_idx_linear = tf.cast(sparse_b.indices[:, 1], tf.int64) - col_idx = utils.unravel_index(col_idx_linear, tf.cast(tensor_shape[1], tf.int64)) - for core_idx in range(tt_a.ndims()): - curr_core = tt_a.tt_cores[core_idx] - left_rank = a_ranks[core_idx] - right_rank = a_ranks[core_idx + 1] - curr_core = tf.transpose(curr_core, (1, 2, 0, 3)) - curr_core_shape = (a_shape[0][core_idx]*a_shape[1][core_idx], left_rank, - right_rank) - curr_core = tf.reshape(curr_core, curr_core_shape) - # Ravel multiindex (row_idx[:, core_idx], col_idx[:, core_idx]) into - # a linear index to use tf.gather that supports only first dimensional - # gather. - # TODO: use gather_nd instead. - curr_elements_idx = row_idx[:, core_idx] * tensor_shape[1][core_idx] - curr_elements_idx += col_idx[:, core_idx] - core_slices = tf.gather(curr_core, curr_elements_idx) - tt_a_elements = tf.matmul(tt_a_elements, core_slices) - else: - tt_a_elements = gather_nd(tt_a, sparse_b.indices) - tt_a_elements = tf.reshape(tt_a_elements, (1, -1)) - sparse_b_elements = tf.reshape(sparse_b.values, (-1, 1)) - result = tf.matmul(tt_a_elements, sparse_b_elements) - # Convert a 1x1 matrix into a number. - result = result[0, 0] - return result - - -def dense_tt_flat_inner(dense_a, tt_b): - """Inner product between a tf.Tensor and TT-tensor (or TT-matrix) along all axis. - - The shapes of dense_a and tt_b should coincide. - - Args: - dense_a: tf.Tensor - tt_b: `TensorTrain` object - - Returns - a number - sum of products of all the elements of dense_a and tt_b - """ - raise NotImplementedError - - -def sparse_tt_flat_inner(sparse_a, tt_b): - """Inner product between a tf.SparseTensor and TT-tensor (or TT-matrix) along all axis. - - The shapes of sparse_a and tt_b should coincide. - - Args: - sparse_a: tf.SparseTensor - tt_b: `TensorTrain` object - - Returns - a number - sum of products of all the elements of sparse_a and tt_b - """ - raise NotImplementedError - - -def flat_inner(a, b): - """Inner product along all axis. - - The shapes of a and b should coincide. - - Args: - a: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor - b: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor - - Returns - a number - sum of products of all the elements of a and b - OR or a tf.Tensor of size batch_size - sum of products of all the elements of a and b for each element in the - batch. - """ -# TODO: is it safe to check types? What if a class is derived from TT? - if isinstance(a, TensorTrainBase) and isinstance(b, TensorTrainBase): - return tt_tt_flat_inner(a, b) - elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): - return tt_dense_flat_inner(a, b) - elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): - return dense_tt_flat_inner(a, b) - elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): - return tt_sparse_flat_inner(a, b) - elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): - return sparse_tt_flat_inner(a, b) - else: - raise ValueError('Argument types are not supported in flat_inner: %s x %s' % - (a, b)) - - -def _add_tensor_cores(tt_a, tt_b): - """Internal function to be called from add for two TT-tensors. - - Does the actual assembling of the TT-cores to add two TT-tensors. - """ - ndims = tt_a.ndims() - dtype = tt_a.dtype - shape = shapes.lazy_raw_shape(tt_a) - a_ranks = shapes.lazy_tt_ranks(tt_a) - b_ranks = shapes.lazy_tt_ranks(tt_b) - tt_cores = [] - for core_idx in range(ndims): - a_core = tt_a.tt_cores[core_idx] - b_core = tt_b.tt_cores[core_idx] - if core_idx == 0: - curr_core = tf.concat((a_core, b_core), axis=2) - elif core_idx == ndims - 1: - curr_core = tf.concat((a_core, b_core), axis=0) - else: - upper_zeros = tf.zeros((a_ranks[core_idx], shape[0][core_idx], - b_ranks[core_idx + 1]), dtype) - lower_zeros = tf.zeros((b_ranks[core_idx], shape[0][core_idx], - a_ranks[core_idx + 1]), dtype) - upper = tf.concat((a_core, upper_zeros), axis=2) - lower = tf.concat((lower_zeros, b_core), axis=2) - curr_core = tf.concat((upper, lower), axis=0) - tt_cores.append(curr_core) - return tt_cores - - -def _add_batch_tensor_cores(tt_a, tt_b): - """Internal function to be called from add for two batches of TT-tensors. - - Does the actual assembling of the TT-cores to add two batches of TT-tensors. - """ - ndims = tt_a.ndims() - dtype = tt_a.dtype - shape = shapes.lazy_raw_shape(tt_a) - a_ranks = shapes.lazy_tt_ranks(tt_a) - b_ranks = shapes.lazy_tt_ranks(tt_b) - if isinstance(tt_a, TensorTrainBatch) and tt_a.batch_size == 1: - # We add 1 element batch tt_a to a batch_size element batch tt_b to get - # the answer TensorTrainBatch of batch_size == tt_b.batch_size. - batch_size = shapes.lazy_batch_size(tt_b) - else: - batch_size = shapes.lazy_batch_size(tt_a) - tt_a = shapes.expand_batch_dim(tt_a) - tt_b = shapes.expand_batch_dim(tt_b) - tt_cores = [] - for core_idx in range(ndims): - a_core = tt_a.tt_cores[core_idx] - if tt_a.batch_size == 1: - a_core = tf.tile(a_core, (batch_size, 1, 1, 1)) - b_core = tt_b.tt_cores[core_idx] - if tt_b.batch_size == 1: - b_core = tf.tile(b_core, (batch_size, 1, 1, 1)) - if core_idx == 0: - curr_core = tf.concat((a_core, b_core), axis=3) - elif core_idx == ndims - 1: - curr_core = tf.concat((a_core, b_core), axis=1) - else: - upper_zeros = tf.zeros((batch_size, a_ranks[core_idx], shape[0][core_idx], - b_ranks[core_idx + 1]), dtype) - lower_zeros = tf.zeros((batch_size, b_ranks[core_idx], shape[0][core_idx], - a_ranks[core_idx + 1]), dtype) - upper = tf.concat((a_core, upper_zeros), axis=3) - lower = tf.concat((lower_zeros, b_core), axis=3) - curr_core = tf.concat((upper, lower), axis=1) - tt_cores.append(curr_core) - return tt_cores, batch_size - - -def _add_matrix_cores(tt_a, tt_b): - """Internal function to be called from add for two TT-matrices. - - Does the actual assembling of the TT-cores to add two TT-matrices. - """ - ndims = tt_a.ndims() - dtype = tt_a.dtype - shape = shapes.lazy_raw_shape(tt_a) - a_ranks = shapes.lazy_tt_ranks(tt_a) - b_ranks = shapes.lazy_tt_ranks(tt_b) - tt_cores = [] - for core_idx in range(ndims): - a_core = tt_a.tt_cores[core_idx] - b_core = tt_b.tt_cores[core_idx] - if core_idx == 0: - curr_core = tf.concat((a_core, b_core), axis=3) - elif core_idx == ndims - 1: - curr_core = tf.concat((a_core, b_core), axis=0) - else: - upper_zeros = tf.zeros((a_ranks[core_idx], shape[0][core_idx], - shape[1][core_idx], b_ranks[core_idx + 1]), dtype) - lower_zeros = tf.zeros((b_ranks[core_idx], shape[0][core_idx], - shape[1][core_idx], a_ranks[core_idx + 1]), dtype) - upper = tf.concat((a_core, upper_zeros), axis=3) - lower = tf.concat((lower_zeros, b_core), axis=3) - curr_core = tf.concat((upper, lower), axis=0) - tt_cores.append(curr_core) - return tt_cores - - -def _add_batch_matrix_cores(tt_a, tt_b): - """Internal function to be called from add for two batches of TT-matrices. - - Does the actual assembling of the TT-cores to add two batches of TT-matrices. - """ - ndims = tt_a.ndims() - dtype = tt_a.dtype - shape = shapes.lazy_raw_shape(tt_a) - a_ranks = shapes.lazy_tt_ranks(tt_a) - b_ranks = shapes.lazy_tt_ranks(tt_b) - if isinstance(tt_a, TensorTrainBatch) and tt_a.batch_size == 1: - # We add 1 element batch tt_a to a batch_size element batch tt_b to get - # the answer TensorTrainBatch of batch_size == tt_b.batch_size. - batch_size = shapes.lazy_batch_size(tt_b) - else: - batch_size = shapes.lazy_batch_size(tt_a) - tt_a = shapes.expand_batch_dim(tt_a) - tt_b = shapes.expand_batch_dim(tt_b) - tt_cores = [] - for core_idx in range(ndims): - a_core = tt_a.tt_cores[core_idx] - if tt_a.batch_size == 1: - a_core = tf.tile(a_core, (batch_size, 1, 1, 1, 1)) - b_core = tt_b.tt_cores[core_idx] - if tt_b.batch_size == 1: - b_core = tf.tile(b_core, (batch_size, 1, 1, 1, 1)) - if core_idx == 0: - curr_core = tf.concat((a_core, b_core), axis=4) - elif core_idx == ndims - 1: - curr_core = tf.concat((a_core, b_core), axis=1) - else: - upper_zeros = tf.zeros((batch_size, a_ranks[core_idx], shape[0][core_idx], - shape[1][core_idx], b_ranks[core_idx + 1]), dtype) - lower_zeros = tf.zeros((batch_size, b_ranks[core_idx], shape[0][core_idx], - shape[1][core_idx], a_ranks[core_idx + 1]), dtype) - upper = tf.concat((a_core, upper_zeros), axis=4) - lower = tf.concat((lower_zeros, b_core), axis=4) - curr_core = tf.concat((upper, lower), axis=1) - tt_cores.append(curr_core) - return tt_cores, batch_size - - -def add(tt_a, tt_b): - """Returns a TensorTrain corresponding to elementwise sum tt_a + tt_b. - - The shapes of tt_a and tt_b should coincide. - Supports broadcasting: - add(TensorTrainBatch, TensorTrain) - adds TensorTrain to each element in the batch of TTs in TensorTrainBatch. - - Args: - tt_a: `TensorTrain`, `TensorTrainBatch`, TT-tensor, or TT-matrix - tt_b: `TensorTrain`, `TensorTrainBatch`, TT-tensor, or TT-matrix - - Returns - a `TensorTrain` object corresponding to the element-wise sum of arguments if - both arguments are `TensorTrain`s. - OR a `TensorTrainBatch` if at least one of the arguments is - `TensorTrainBatch` - - Raises - ValueError if the arguments shapes do not coincide - """ - ndims = tt_a.ndims() - if tt_a.is_tt_matrix() != tt_b.is_tt_matrix(): - raise ValueError('The arguments should be both TT-tensors or both ' - 'TT-matrices') - - if tt_a.get_raw_shape() != tt_b.get_raw_shape(): - raise ValueError('The arguments should have the same shape.') - - if not shapes.is_batch_broadcasting_possible(tt_a, tt_b): - raise ValueError('The batch sizes are different and not 1, broadcasting is ' - 'not available.') - - is_batch_case = isinstance(tt_a, TensorTrainBatch) or isinstance(tt_b, TensorTrainBatch) - batch_size = None - if is_batch_case: - if tt_a.is_tt_matrix(): - tt_cores, batch_size = _add_batch_matrix_cores(tt_a, tt_b) - else: - tt_cores, batch_size = _add_batch_tensor_cores(tt_a, tt_b) - else: - if tt_a.is_tt_matrix(): - tt_cores = _add_matrix_cores(tt_a, tt_b) - else: - tt_cores = _add_tensor_cores(tt_a, tt_b) - - out_ranks = [1] - static_a_ranks = tt_a.get_tt_ranks() - static_b_ranks = tt_b.get_tt_ranks() - for core_idx in range(1, ndims): - out_ranks.append(static_a_ranks[core_idx] + static_b_ranks[core_idx]) - out_ranks.append(1) - if is_batch_case: - return TensorTrainBatch(tt_cores, tt_a.get_raw_shape(), out_ranks, - batch_size) - else: - return TensorTrain(tt_cores, tt_a.get_raw_shape(), out_ranks) - - -def multiply(tt_left, right): - """Returns a TensorTrain corresponding to element-wise product tt_left * right. - - Supports broadcasting: - multiply(TensorTrainBatch, TensorTrain) returns TensorTrainBatch consisting - of element-wise products of TT in TensorTrainBatch and TensorTrain - - multiply(TensorTrainBatch_a, TensorTrainBatch_b) returns TensorTrainBatch - consisting of element-wise products of TT in TensorTrainBatch_a and - TT in TensorTrainBatch_b - - Batch sizes should support broadcasting - Args: - tt_left: `TensorTrain` OR `TensorTrainBatch` - right: `TensorTrain` OR `TensorTrainBatch` OR a number. - - Returns - a `TensorTrain` or `TensorTrainBatch` object corresponding to the - element-wise product of the arguments. - - Raises - ValueError if the arguments shapes do not coincide or broadcasting is not - possible. - """ - - is_left_batch = isinstance(tt_left, TensorTrainBatch) - is_right_batch = isinstance(right, TensorTrainBatch) - - is_batch_case = is_left_batch or is_right_batch - ndims = tt_left.ndims() - if not isinstance(right, TensorTrainBase): - # Assume right is a number, not TensorTrain. - # To squash right uniformly across TT-cores we pull its absolute value - # and raise to the power 1/ndims. First TT-core is multiplied by the sign - # of right. - tt_cores = list(tt_left.tt_cores) - fact = tf.pow(tf.cast(tf.abs(right), tt_left.dtype), 1.0 / ndims) - sign = tf.cast(tf.sign(right), tt_left.dtype) - for i in range(len(tt_cores)): - tt_cores[i] = fact * tt_cores[i] - - tt_cores[0] = tt_cores[0] * sign - out_ranks = tt_left.get_tt_ranks() - if is_left_batch: - out_batch_size = tt_left.batch_size - else: - - if tt_left.is_tt_matrix() != right.is_tt_matrix(): - raise ValueError('The arguments should be both TT-tensors or both ' - 'TT-matrices') - - if tt_left.get_raw_shape() != right.get_raw_shape(): - raise ValueError('The arguments should have the same shape.') - - out_batch_size = 1 - dependencies = [] - can_determine_if_broadcast = True - if is_left_batch and is_right_batch: - if tt_left.batch_size is None and right.batch_size is None: - can_determine_if_broadcast = False - elif tt_left.batch_size is None and right.batch_size is not None: - if right.batch_size > 1: - can_determine_if_broadcast = False - elif tt_left.batch_size is not None and right.batch_size is None: - if tt_left.batch_size > 1: - can_determine_if_broadcast = False - - if not can_determine_if_broadcast: - # Cannot determine if broadcasting is needed. Avoid broadcasting and - # assume elementwise multiplication AND add execution time assert to print - # a better error message if the batch sizes turn out to be different. - - message = ('The batch sizes were unknown on compilation stage, so ' - 'assumed elementwise multiplication (i.e. no broadcasting). ' - 'Now it seems that they are different after all :') - - data = [message, shapes.lazy_batch_size(tt_left), ' x ', - shapes.lazy_batch_size(right)] - bs_eq = tf.assert_equal(shapes.lazy_batch_size(tt_left), - shapes.lazy_batch_size(right), data=data) - - dependencies.append(bs_eq) - - do_broadcast = shapes.is_batch_broadcasting_possible(tt_left, right) - if not can_determine_if_broadcast: - # Assume elementwise multiplication if broadcasting cannot be determined - # on compilation stage. - do_broadcast = False - if not do_broadcast and can_determine_if_broadcast: - raise ValueError('The batch sizes are different and not 1, broadcasting ' - 'is not available.') - - a_ranks = shapes.lazy_tt_ranks(tt_left) - b_ranks = shapes.lazy_tt_ranks(right) - shape = shapes.lazy_raw_shape(tt_left) - - output_str = '' - bs_str_left = '' - bs_str_right = '' - - if is_batch_case: - if is_left_batch and is_right_batch: - # Both arguments are batches of equal size. - if tt_left.batch_size == right.batch_size or not can_determine_if_broadcast: - bs_str_left = 'n' - bs_str_right = 'n' - output_str = 'n' - if not can_determine_if_broadcast: - out_batch_size = None - else: - out_batch_size = tt_left.batch_size - else: - # Broadcasting (e.g batch_sizes are 1 and n>1). - bs_str_left = 'n' - bs_str_right = 'm' - output_str = 'nm' - if tt_left.batch_size is None or tt_left.batch_size > 1: - out_batch_size = tt_left.batch_size - else: - out_batch_size = right.batch_size - else: - # One of the arguments is TensorTrain. - if is_left_batch: - bs_str_left = 'n' - bs_str_right = '' - out_batch_size = tt_left.batch_size - else: - bs_str_left = '' - bs_str_right = 'n' - out_batch_size = right.batch_size - output_str = 'n' - - is_matrix = tt_left.is_tt_matrix() - tt_cores = [] - - for core_idx in range(ndims): - a_core = tt_left.tt_cores[core_idx] - b_core = right.tt_cores[core_idx] - left_rank = a_ranks[core_idx] * b_ranks[core_idx] - right_rank = a_ranks[core_idx + 1] * b_ranks[core_idx + 1] - if is_matrix: - with tf.control_dependencies(dependencies): - curr_core = tf.einsum('{0}aijb,{1}cijd->{2}acijbd'.format(bs_str_left, - bs_str_right, output_str), a_core, b_core) - curr_core = tf.reshape(curr_core, (-1, left_rank, - shape[0][core_idx], - shape[1][core_idx], - right_rank)) - if not is_batch_case: - curr_core = tf.squeeze(curr_core, axis=0) - else: - with tf.control_dependencies(dependencies): - curr_core = tf.einsum('{0}aib,{1}cid->{2}acibd'.format(bs_str_left, - bs_str_right, output_str), a_core, b_core) - curr_core = tf.reshape(curr_core, (-1, left_rank, - shape[0][core_idx], right_rank)) - if not is_batch_case: - curr_core = tf.squeeze(curr_core, axis=0) - - tt_cores.append(curr_core) - - combined_ranks = zip(tt_left.get_tt_ranks(), right.get_tt_ranks()) - out_ranks = [a * b for a, b in combined_ranks] - - if not is_batch_case: - return TensorTrain(tt_cores, tt_left.get_raw_shape(), out_ranks) - else: - return TensorTrainBatch(tt_cores, tt_left.get_raw_shape(), out_ranks, - batch_size=out_batch_size) - -def frobenius_norm_squared(tt, differentiable=False): - """Frobenius norm squared of `TensorTrain` or of each TT in `TensorTrainBatch`. - - Frobenius norm squared is the sum of squares of all elements in a tensor. - - Args: - tt: `TensorTrain` or `TensorTrainBatch` object - differentiable: bool, whether to use a differentiable implementation - or a fast and stable implementation based on QR decomposition. - - Returns - a number which is the Frobenius norm squared of `tt`, if it is `TensorTrain` - OR - a Tensor of size tt.batch_size, consisting of the Frobenius norms squared of - each TensorTrain in `tt`, if it is `TensorTrainBatch` - """ - if differentiable: - if hasattr(tt, 'batch_size'): - bs_str = 'n' - else: - bs_str = '' - if tt.is_tt_matrix(): - running_prod = tf.einsum('{0}aijb,{0}cijd->{0}bd'.format(bs_str), - tt.tt_cores[0], tt.tt_cores[0]) - else: - running_prod = tf.einsum('{0}aib,{0}cid->{0}bd'.format(bs_str), - tt.tt_cores[0], tt.tt_cores[0]) - - for core_idx in range(1, tt.ndims()): - curr_core = tt.tt_cores[core_idx] - if tt.is_tt_matrix(): - running_prod = tf.einsum('{0}ac,{0}aijb,{0}cijd->{0}bd'.format(bs_str), - running_prod, curr_core, curr_core) - else: - running_prod = tf.einsum('{0}ac,{0}aib,{0}cid->{0}bd'.format(bs_str), - running_prod, curr_core, curr_core) - - return tf.squeeze(running_prod, [-1, -2]) - - else: - orth_tt = decompositions.orthogonalize_tt_cores(tt, left_to_right=True) - # All the cores of orth_tt except the last one are orthogonal, hence - # the Frobenius norm of orth_tt equals to the norm of the last core. - if hasattr(tt, 'batch_size'): - batch_size = shapes.lazy_batch_size(tt) - last_core = tf.reshape(orth_tt.tt_cores[-1], (batch_size, -1)) - return tf.norm(last_core, axis=1) ** 2 - else: - return tf.norm(orth_tt.tt_cores[-1]) ** 2 - - -def frobenius_norm(tt, epsilon=1e-5, differentiable=False): - """Frobenius norm of `TensorTrain` or of each TT in `TensorTrainBatch` - - Frobenius norm is the sqrt of the sum of squares of all elements in a tensor. - - Args: - tt: `TensorTrain` or `TensorTrainBatch` object - epsilon: the function actually computes sqrt(norm_squared + epsilon) for - numerical stability (e.g. gradient of sqrt at zero is inf). - differentiable: bool, whether to use a differentiable implementation or - a fast and stable implementation based on QR decomposition. - - Returns - a number which is the Frobenius norm of `tt`, if it is `TensorTrain` - OR - a Tensor of size tt.batch_size, consisting of the Frobenius norms of - each TensorTrain in `tt`, if it is `TensorTrainBatch` - """ - return tf.sqrt(frobenius_norm_squared(tt, differentiable) + epsilon) - - -def transpose(tt_matrix): - """Transpose a TT-matrix or a batch of TT-matrices. - - Args: - tt_matrix: `TensorTrain` or `TensorTrainBatch` object containing a TT-matrix - (or a batch of TT-matrices). - - Returns: - `TensorTrain` or `TensorTrainBatch` object containing a transposed TT-matrix - (or a batch of TT-matrices). - - Raises: - ValueError if the argument is not a TT-matrix. - """ - if not isinstance(tt_matrix, TensorTrainBase) or not tt_matrix.is_tt_matrix(): - raise ValueError('The argument should be a TT-matrix.') - - transposed_tt_cores = [] - for core_idx in range(tt_matrix.ndims()): - curr_core = tt_matrix.tt_cores[core_idx] - if isinstance(tt_matrix, TensorTrain): - transposed_tt_cores.append(tf.transpose(curr_core, (0, 2, 1, 3))) - else: - # TensorTrainBatch. - transposed_tt_cores.append(tf.transpose(curr_core, (0, 1, 3, 2, 4))) - - tt_matrix_shape = tt_matrix.get_raw_shape() - transposed_shape = tt_matrix_shape[1], tt_matrix_shape[0] - tt_ranks = tt_matrix.get_tt_ranks() - if isinstance(tt_matrix, TensorTrain): - return TensorTrain(transposed_tt_cores, transposed_shape, tt_ranks) - else: - batch_size = tt_matrix.batch_size - return TensorTrainBatch(transposed_tt_cores, transposed_shape, tt_ranks, - batch_size) - - -def quadratic_form(A, b, c): - """Quadratic form b^t A c; A is a TT-matrix, b and c can be batches. - - Args: - A: `TensorTrain` object containing a TT-matrix of size N x M. - b: `TensorTrain` object containing a TT-matrix of size N x 1 - or `TensorTrainBatch` with a batch of TT-matrices of size N x 1. - c: `TensorTrain` object containing a TT-matrix of size M x 1 - or `TensorTrainBatch` with a batch of TT-matrices of size M x 1. - - Returns: - A number, the value of the quadratic form if all the arguments are - `TensorTrain`s. - OR tf.Tensor of size batch_size if at least one of the arguments is - `TensorTrainBatch` - - Raises: - ValueError if the arguments are not TT-matrices or if the shapes are - not consistent. - """ - if not isinstance(A, TensorTrainBase) or not A.is_tt_matrix(): - raise ValueError('The arguments should be a TT-matrix.') - - # TODO: support tf.Tensor as b and c. - if not isinstance(b, TensorTrainBase) or not b.is_tt_matrix(): - raise ValueError('The arguments should be a TT-matrix.') - if not isinstance(c, TensorTrainBase) or not c.is_tt_matrix(): - raise ValueError('The arguments should be a TT-matrix.') - - b_is_batch = isinstance(b, TensorTrainBatch) - c_is_batch = isinstance(b, TensorTrainBatch) - b_bs_str = 'p' if b_is_batch else '' - c_bs_str = 'p' if c_is_batch else '' - out_bs_str = 'p' if b_is_batch or c_is_batch else '' - - ndims = A.ndims() - curr_core_1 = b.tt_cores[0] - curr_core_2 = c.tt_cores[0] - curr_matrix_core = A.tt_cores[0] - # We enumerate the dummy dimension (that takes 1 value) with `k`. - # You may think that using two different k would be faster, but in my - # experience it's even a little bit slower (but neglectable in general). - einsum_str = '{0}aikb,cijd,{1}ejkf->{2}bdf'.format(b_bs_str, c_bs_str, - out_bs_str) - res = tf.einsum(einsum_str, curr_core_1, curr_matrix_core, curr_core_2) - for core_idx in range(1, ndims): - curr_core_1 = b.tt_cores[core_idx] - curr_core_2 = c.tt_cores[core_idx] - curr_matrix_core = A.tt_cores[core_idx] - einsum_str = '{2}ace,{0}aikb,cijd,{1}ejkf->{2}bdf'.format(b_bs_str, - c_bs_str, - out_bs_str) - res = tf.einsum(einsum_str, res, curr_core_1, - curr_matrix_core, curr_core_2) - - # Squeeze to make the result a number instead of 1 x 1 for NON batch case and - # to make the result a tensor of size - # batch_size - # instead of - # batch_size x 1 x 1 - # in the batch case. - return tf.squeeze(res) - - -def cast(tt_a, dtype): - """Casts a tt-tensor to a new type. - - Args: - tt_a: `TensorTrain` object. - dtype: The destination type. - - Raises: - TypeError: If `tt_a` cannot be cast to the `dtype`. - ValueError: If `tt_a` is not a `TensorTrain` or `TensorTrainBatch`. - """ - res_cores = [] - cores = tt_a.tt_cores - for core_idx in range(tt_a.ndims()): - res_cores.append(tf.cast(cores[core_idx], dtype)) - res_shape = tt_a.get_raw_shape() - res_ranks = tt_a.get_tt_ranks() - if isinstance(tt_a, TensorTrain): - return TensorTrain(res_cores, res_shape, res_ranks) - elif isinstance(tt_a, TensorTrainBatch): - return TensorTrainBatch(res_cores, res_shape, res_ranks, tt_a.batch_size) - else: - raise ValueError('Unsupported type of input "%s", should be TensorTrain or ' - 'TensorTrainBatch.' % tt_a) - - -def gather_nd(tt, indices): - """out[i] = tt[indices[i, 0], indices[i, 1], ...] - - Equivalent to - tf.gather_nd(t3f.full(tt), indices) - but much faster, since it does not materialize the full tensor. - - For batches of TT works indices should include the batch dimension as well. - - Args: - tt: `TensorTrain` or `TensorTrainBatch` object representing a tensor - (TT-matrices are not implemented yet) - indices: numpy array, tf.Tensor, placeholder with 2 or more dimensions. - The last dimension indices.shape[-1] should be equal to the numbers of - dimensions in TT: - indices.shape[-1] = tt.ndims for `TensorTrain` - indices.shape[-1] = tt.ndims + 1 for `TensorTrainBatch` - - Returns: - tf.Tensor with elements specified by indices. - - Raises: - ValueError if `indices` have wrong shape. - NotImplementedError if `tt` is a TT-matrix. - """ - if tt.is_tt_matrix(): - raise NotImplementedError('gather_nd doesnt support TT-matrices yet ' - '(got %s)' % tt) - indices = tf.convert_to_tensor(indices) - if isinstance(tt, TensorTrainBatch): - if indices.get_shape()[-1] != tt.ndims() + 1: - raise ValueError('The last dimension of indices (%d) should have ' - 'the same size as the number of dimensions in the tt ' - 'object (%d) + 1 (for the batch dimension).' % - (indices.get_shape()[-1], tt.ndims())) - else: - if indices.get_shape()[-1] != tt.ndims(): - raise ValueError('The last dimension of indices (%d) should have ' - 'the same size as the number of dimensions in the tt ' - 'object (%d).' % (indices.get_shape()[-1], tt.ndims())) - tt_elements = tf.ones(tf.shape(indices)[:-1]) - tt_elements = tf.reshape(tt_elements, (-1, 1, 1)) - for core_idx in range(tt.ndims()): - curr_core = tt.tt_cores[core_idx] - if isinstance(tt, TensorTrainBatch): - curr_core = tf.transpose(curr_core, (0, 2, 1, 3)) - curr_idx = tf.stack((indices[:, 0], indices[:, core_idx + 1]), axis=1) - core_slices = tf.gather_nd(curr_core, curr_idx) - else: - curr_core = tf.transpose(curr_core, (1, 0, 2)) - core_slices = tf.gather(curr_core, indices[:, core_idx]) - tt_elements = tf.matmul(tt_elements, core_slices) - tt_elements = tf.reshape(tt_elements, tf.shape(indices)[:-1]) - return tt_elements - - -def renormalize_tt_cores(tt, epsilon=1e-8): - """Renormalizes TT-cores to make them of the same Frobenius norm. - - Doesn't change the tensor represented by `tt` object, but renormalizes the - TT-cores to make further computations more stable. - - Args: - tt: `TensorTrain` or `TensorTrainBatch` object - epsilon: parameter for numerical stability of sqrt - Returns: - `TensorTrain` or `TensorTrainBatch` which represents the same - tensor as tt, but with all cores having equal norm. In the batch - case applies to each TT in `TensorTrainBatch`. - - """ - if isinstance(tt, TensorTrain): - new_cores = [] - running_log_norm = 0 - core_norms = [] - for core in tt.tt_cores: - cur_core_norm = tf.sqrt(tf.maximum(tf.reduce_sum(core ** 2), epsilon)) - core_norms.append(cur_core_norm) - running_log_norm += tf.log(cur_core_norm) - - running_log_norm = running_log_norm / tt.ndims() - fact = tf.exp(running_log_norm) - for i, core in enumerate(tt.tt_cores): - new_cores.append(core * fact / core_norms[i]) - - return TensorTrain(new_cores) - else: - sz = (tt.batch_size,) + (len(tt.tt_cores[0].shape) - 1) * (1,) - running_core_log_norms = tf.zeros(sz) - ax = np.arange(len(tt.tt_cores[0].shape))[1:] - fact_list = [] - for core in tt.tt_cores: - cur_core_norm_sq = tf.reduce_sum(core**2, axis=ax, keep_dims=True) - cur_core_norm = tf.sqrt(tf.maximum(epsilon, cur_core_norm_sq)) - fact_list.append(cur_core_norm) - running_core_log_norms += tf.log(cur_core_norm) - - new_cores = [] - exp_fact = tf.exp(running_core_log_norms / tt.ndims()) - for i, core in enumerate(tt.tt_cores): - new_cores.append(tf.multiply(core, exp_fact / fact_list[i])) - - return TensorTrainBatch(new_cores) diff --git a/build/lib/t3f/ops_test.py b/build/lib/t3f/ops_test.py deleted file mode 100644 index cb1dcc7c..00000000 --- a/build/lib/t3f/ops_test.py +++ /dev/null @@ -1,880 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f.tensor_train import TensorTrain -from t3f.tensor_train_batch import TensorTrainBatch -from t3f import ops -from t3f import shapes -from t3f import initializers - - -class TTTensorTest(tf.test.TestCase): - - def testFullTensor2d(self): - np.random.seed(1) - for rank in [1, 2]: - a = np.random.rand(10, rank).astype(np.float32) - b = np.random.rand(rank, 9).astype(np.float32) - tt_cores = (a.reshape(1, 10, rank), b.reshape(rank, 9, 1)) - desired = np.dot(a, b) - with self.test_session(): - tf_tens = TensorTrain(tt_cores) - actual = ops.full(tf_tens) - self.assertAllClose(desired, actual.eval()) - - def testFullTensor3d(self): - np.random.seed(1) - for rank_1 in [1, 2]: - a = np.random.rand(10, rank_1).astype(np.float32) - b = np.random.rand(rank_1, 9, 3).astype(np.float32) - c = np.random.rand(3, 8).astype(np.float32) - tt_cores = (a.reshape(1, 10, rank_1), b, c.reshape((3, 8, 1))) - # Basically do full by hand. - desired = a.dot(b.reshape((rank_1, -1))) - desired = desired.reshape((-1, 3)).dot(c) - desired = desired.reshape(10, 9, 8) - with self.test_session(): - tf_tens = TensorTrain(tt_cores) - actual = ops.full(tf_tens) - self.assertAllClose(desired, actual.eval()) - - def testFlatInnerTTTensbyTTTens(self): - # Inner product between two TT-tensors. - shape_list = ((2, 2), - (2, 3, 4), - (4, 2, 5, 2)) - rank_list = (1, 2) - with self.test_session() as sess: - for shape in shape_list: - for rank in rank_list: - tt_1 = initializers.random_tensor(shape, tt_rank=rank) - tt_2 = initializers.random_tensor(shape, tt_rank=rank) - res_actual = ops.flat_inner(tt_1, tt_2) - tt_1_full = tf.reshape(ops.full(tt_1), (1, -1)) - tt_2_full = tf.reshape(ops.full(tt_2), (-1, 1)) - res_desired = tf.matmul(tt_1_full, tt_2_full) - res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) - self.assertAllClose(res_actual_val, np.squeeze(res_desired_val), - rtol=1e-5) - - def testFlatInnerTTTensbySparseTens(self): - # Inner product between a TT-tensor and a sparse tensor. - shape_list = ((2, 2), - (2, 3, 4), - (4, 2, 5, 2)) - rank_list = (1, 2) - np.random.seed(1) - with self.test_session() as sess: - for shape in shape_list: - for rank in rank_list: - for num_elements in [1, 10]: - tt_1 = initializers.random_tensor(shape, tt_rank=rank) - sparse_flat_indices = np.random.choice(np.prod(shape), num_elements).astype(int) - sparse_indices = np.unravel_index(sparse_flat_indices, shape) - sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(num_elements).astype(np.float32) - sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, - dense_shape=shape) - res_actual = ops.flat_inner(tt_1, sparse_2) - res_actual_val, tt_1_val = sess.run([res_actual, ops.full(tt_1)]) - res_desired_val = tt_1_val.flatten()[sparse_flat_indices].dot(values) - self.assertAllClose(res_actual_val, res_desired_val) - - def testAdd(self): - # Sum two TT-tensors. - tt_a = initializers.random_tensor((2, 1, 3, 4), tt_rank=2) - tt_b = initializers.random_tensor((2, 1, 3, 4), tt_rank=[1, 2, 4, 3, 1]) - with self.test_session() as sess: - res_actual = ops.full(ops.add(tt_a, tt_b)) - res_actual2 = ops.full(tt_a + tt_b) - res_desired = ops.full(tt_a) + ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testMultiply(self): - # Multiply two TT-tensors. - tt_a = initializers.random_tensor((1, 2, 3, 4), tt_rank=2) - tt_b = initializers.random_tensor((1, 2, 3, 4), tt_rank=[1, 1, 4, 3, 1]) - with self.test_session() as sess: - res_actual = ops.full(ops.multiply(tt_a, tt_b)) - res_actual2 = ops.full(tt_a * tt_b) - res_desired = ops.full(tt_a) * ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testMultiplyByNumber(self): - # Multiply a tensor by a number. - tt = initializers.random_tensor((1, 2, 3), tt_rank=(1, 2, 3, 1)) - with self.test_session() as sess: - res_actual = ops.full(ops.multiply(tt, 4)) - res_actual2 = ops.full(4.0 * tt) - res_desired = 4.0 * ops.full(tt) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testFrobeniusNormTens(self): - # Frobenius norm of a TT-tensor. - shape_list = ((2, 2), - (2, 3, 4), - (4, 2, 5, 2)) - rank_list = (1, 2) - with self.test_session() as sess: - for shape in shape_list: - for rank in rank_list: - tt = initializers.random_tensor(shape, tt_rank=rank) - norm_sq_actual = ops.frobenius_norm_squared(tt) - norm_actual = ops.frobenius_norm(tt) - vars = [norm_sq_actual, norm_actual, ops.full(tt)] - norm_sq_actual_val, norm_actual_val, tt_val = sess.run(vars) - tt_val = tt_val.flatten() - norm_sq_desired_val = tt_val.dot(tt_val) - norm_desired_val = np.linalg.norm(tt_val) - self.assertAllClose(norm_sq_actual_val, norm_sq_desired_val) - self.assertAllClose(norm_actual_val, norm_desired_val, atol=1e-5, - rtol=1e-5) - - def testCastFloat(self): - # Test cast function for float tt-tensors. - tt_x = initializers.random_tensor((2, 3, 2), tt_rank=2) - - with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_x, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) - - def testCastIntFloat(self): - # Tests cast function from int to float for tensors. - np.random.seed(1) - K_1 = np.random.randint(0, high=100, size=(1, 2, 2)) - K_2 = np.random.randint(0, high=100, size=(2, 3, 2)) - K_3 = np.random.randint(0, high=100, size=(2, 2, 1)) - tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) - - with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_int, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) - - def testCoreRenorm(self): - a = initializers.random_tensor(3 * (10,), tt_rank=7) - b = ops.renormalize_tt_cores(a) - var_list = [ops.full(a), ops.full(b)] - with self.test_session() as sess: - af, bf = sess.run(var_list) - b_cores = sess.run(b.tt_cores) - b_cores_norms = [] - for cr in b_cores: - b_cores_norms.append(np.linalg.norm(cr)) - self.assertAllClose(af, bf, atol=1e-5, rtol=1e-5) - self.assertAllClose(b_cores_norms, b_cores_norms[0] - * np.ones((len(b_cores)))) - - -class TTMatrixTest(tf.test.TestCase): - - def testFullMatrix2d(self): - np.random.seed(1) - for rank in [1, 2]: - a = np.random.rand(2, 3, rank).astype(np.float32) - b = np.random.rand(rank, 4, 5).astype(np.float32) - tt_cores = (a.reshape(1, 2, 3, rank), b.reshape((rank, 4, 5, 1))) - # Basically do full by hand. - desired = a.reshape((-1, rank)).dot(b.reshape((rank, -1))) - desired = desired.reshape((2, 3, 4, 5)) - desired = desired.transpose((0, 2, 1, 3)) - desired = desired.reshape((2 * 4, 3 * 5)) - with self.test_session(): - tf_mat = TensorTrain(tt_cores) - actual = ops.full(tf_mat) - self.assertAllClose(desired, actual.eval()) - - def testFullMatrix3d(self): - np.random.seed(1) - for rank in [1, 2]: - a = np.random.rand(2, 3, rank).astype(np.float32) - b = np.random.rand(rank, 4, 5, rank).astype(np.float32) - c = np.random.rand(rank, 2, 2).astype(np.float32) - tt_cores = (a.reshape(1, 2, 3, rank), b.reshape(rank, 4, 5, rank), - c.reshape(rank, 2, 2, 1)) - # Basically do full by hand. - desired = a.reshape((-1, rank)).dot(b.reshape((rank, -1))) - desired = desired.reshape((-1, rank)).dot(c.reshape((rank, -1))) - desired = desired.reshape((2, 3, 4, 5, 2, 2)) - desired = desired.transpose((0, 2, 4, 1, 3, 5)) - desired = desired.reshape((2 * 4 * 2, 3 * 5 * 2)) - with self.test_session(): - tf_mat = TensorTrain(tt_cores) - actual = ops.full(tf_mat) - self.assertAllClose(desired, actual.eval()) - - def testTTMatTimesTTMat(self): - # Multiply a TT-matrix by another TT-matrix. - left_shape = (2, 3, 4) - sum_shape = (4, 3, 5) - right_shape = (4, 4, 4) - with self.test_session() as sess: - tt_mat_1 = initializers.random_matrix((left_shape, sum_shape), tt_rank=3) - tt_mat_2 = initializers.random_matrix((sum_shape, right_shape)) - res_actual = ops.matmul(tt_mat_1, tt_mat_2) - res_actual = ops.full(res_actual) - res_desired = tf.matmul(ops.full(tt_mat_1), ops.full(tt_mat_2)) - res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) - # TODO: why so bad accuracy? - self.assertAllClose(res_actual_val, res_desired_val, atol=1e-4, rtol=1e-4) - - def testTTMatTimesDenseVec(self): - # Multiply a TT-matrix by a dense vector. - inp_shape = (2, 3, 4) - out_shape = (3, 4, 3) - np.random.seed(1) - vec = np.random.rand(np.prod(inp_shape), 1).astype(np.float32) - with self.test_session() as sess: - tf_vec = tf.constant(vec) - tf.set_random_seed(1) - tt_mat = initializers.random_matrix((out_shape, inp_shape)) - res_actual = ops.matmul(tt_mat, tf_vec) - res_desired = tf.matmul(ops.full(tt_mat), tf_vec) - res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) - self.assertAllClose(res_actual_val, res_desired_val) - - def testDenseMatTimesTTVec(self): - # Multiply a TT-matrix by a dense vector. - inp_shape = (3, 3, 3, 3) - out_shape = (3, 3, 3, 3) - np.random.seed(1) - mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)).astype(np.float32) - with self.test_session() as sess: - tf_mat = tf.constant(mat) - tf.set_random_seed(1) - tt_vec = initializers.random_matrix((inp_shape, None)) - res_actual = ops.matmul(tf_mat, tt_vec) - res_desired = tf.matmul(tf_mat, ops.full(tt_vec)) - res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) - self.assertAllClose(res_actual_val, res_desired_val, atol=1e-4, rtol=1e-4) - - def testFlatInnerTTMatbyTTMat(self): - # Inner product between two TT-Matrices. - shape_list = (((2, 2), (3, 4)), - ((2, 3, 4), (2, 2, 2))) - rank_list = (1, 2) - with self.test_session() as sess: - for shape in shape_list: - for rank in rank_list: - tt_1 = initializers.random_matrix(shape, tt_rank=rank) - tt_2 = initializers.random_matrix(shape, tt_rank=rank) - res_actual = ops.flat_inner(tt_1, tt_2) - tt_1_full = tf.reshape(ops.full(tt_1), (1, -1)) - tt_2_full = tf.reshape(ops.full(tt_2), (-1, 1)) - res_desired = tf.matmul(tt_1_full, tt_2_full) - res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) - self.assertAllClose(res_actual_val, np.squeeze(res_desired_val), - rtol=1e-5, atol=1e-5) - - def testFlatInnerTTMatbySparseMat(self): - # Inner product between a TT-matrix and a sparse matrix. - shape_list = (((2, 2), (3, 4)), - ((2, 3, 4), (2, 2, 2))) - rank_list = (1, 2) - np.random.seed(1) - with self.test_session() as sess: - for tensor_shape in shape_list: - for rank in rank_list: - for num_elements in [1, 9]: - tt_1 = initializers.random_matrix(tensor_shape, tt_rank=rank) - matrix_shape = np.prod(tensor_shape[0]), np.prod(tensor_shape[1]) - sparse_flat_indices = np.random.choice(np.prod(matrix_shape), num_elements) - sparse_flat_indices = sparse_flat_indices.astype(int) - sparse_indices = np.unravel_index(sparse_flat_indices, matrix_shape) - sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(num_elements).astype(np.float32) - sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, - dense_shape=matrix_shape) - res_actual = ops.flat_inner(tt_1, sparse_2) - res_actual_val, tt_1_val = sess.run([res_actual, ops.full(tt_1)]) - res_desired_val = tt_1_val.flatten()[sparse_flat_indices].dot(values) - self.assertAllClose(res_actual_val, res_desired_val) - - def testFrobeniusNormMatrix(self): - # Frobenius norm of a TT-matrix. - shape_list = (((2, 2), (3, 4)), - ((2, 3, 4), (2, 2, 2))) - rank_list = (1, 2) - with self.test_session() as sess: - for tensor_shape in shape_list: - for rank in rank_list: - tt = initializers.random_matrix(tensor_shape, tt_rank=rank) - norm_sq_actual = ops.frobenius_norm_squared(tt) - norm_actual = ops.frobenius_norm(tt) - vars = [norm_sq_actual, norm_actual, ops.full(tt)] - norm_sq_actual_val, norm_actual_val, tt_val = sess.run(vars) - tt_val = tt_val.flatten() - norm_sq_desired_val = tt_val.dot(tt_val) - norm_desired_val = np.linalg.norm(tt_val) - self.assertAllClose(norm_sq_actual_val, norm_sq_desired_val) - self.assertAllClose(norm_actual_val, norm_desired_val, atol=1e-5, - rtol=1e-5) - - def testTranspose(self): - # Transpose a TT-matrix. - shape_list = (((2, 2), (3, 4)), - ((2, 3, 4), (2, 2, 2))) - rank_list = (1, 2) - with self.test_session() as sess: - for tensor_shape in shape_list: - for rank in rank_list: - tt = initializers.random_matrix(tensor_shape, tt_rank=rank) - res_actual = ops.full(ops.transpose(tt)) - res_actual_val, tt_val = sess.run([res_actual, ops.full(tt)]) - self.assertAllClose(tt_val.transpose(), res_actual_val) - - def testQuadraticForm(self): - # Test quadratic form. - shape_list = (((2, 2), (3, 4)), - ((2, 3, 4), (2, 2, 2))) - rank_list = (1, 2) - with self.test_session() as sess: - for tensor_shape in shape_list: - for rank in rank_list: - A = initializers.random_matrix(tensor_shape, tt_rank=rank) - b = initializers.random_matrix((tensor_shape[0], None), tt_rank=rank) - c = initializers.random_matrix((tensor_shape[1], None), tt_rank=rank) - res_actual = ops.quadratic_form(A, b, c) - vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] - res_actual_val, A_val, b_val, c_val = sess.run(vars) - res_desired = b_val.T.dot(A_val).dot(c_val) - self.assertAllClose(res_actual_val, np.squeeze(res_desired), - atol=1e-5, rtol=1e-5) - - def testQuadraticFormBatch(self): - # Test quadratic form for batch of tensors. - shape_list = (((2, 2), (3, 4)), - ((2, 3, 4), (2, 2, 2))) - rank_list = (1, 2) - with self.test_session() as sess: - for tensor_shape in shape_list: - for rank in rank_list: - A = initializers.random_matrix(tensor_shape, tt_rank=rank) - b = initializers.random_matrix_batch((tensor_shape[0], None), - tt_rank=rank, batch_size=5) - c = initializers.random_matrix_batch((tensor_shape[1], None), - tt_rank=rank, batch_size=5) - res_actual = ops.quadratic_form(A, b, c) - vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] - res_actual_val, A_val, b_val, c_val = sess.run(vars) - res_desired = np.diag(b_val[:, :, 0].dot(A_val).dot(c_val[:, :, 0].T)) - self.assertAllClose(res_actual_val, np.squeeze(res_desired), - atol=1e-5, rtol=1e-5) - - def testCastFloat(self): - # Test cast function for float tt-matrices and vectors. - - tt_mat = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=2) - tt_vec = initializers.random_matrix(((2, 3), None), tt_rank=2) - - with self.test_session() as sess: - for tt in [tt_mat, tt_vec]: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) - - def testCastIntFloat(self): - # Tests cast function from int to float for matrices. - np.random.seed(1) - K_1 = np.random.randint(0, high=100, size=(1, 2, 2, 2)) - K_2 = np.random.randint(0, high=100, size=(2, 3, 3, 2)) - K_3 = np.random.randint(0, high=100, size=(2, 2, 2, 1)) - tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) - - with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_int, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) - - def testUnknownRanksTTMatmul(self): - # Tests tt_tt_matmul for matrices with unknown ranks - K_1 = tf.placeholder(tf.float32, (1, 2, 2, None)) - K_2 = tf.placeholder(tf.float32, (None, 3, 3, 1)) - tt_mat = TensorTrain([K_1, K_2]) - res_actual = ops.full(ops.matmul(tt_mat, tt_mat)) - res_desired = tf.matmul(ops.full(tt_mat), ops.full(tt_mat)) - np.random.seed(1) - K_1_val = np.random.rand(1, 2, 2, 2) - K_2_val = np.random.rand(2, 3, 3, 1) - with self.test_session() as sess: - res_actual_val = sess.run(res_actual, {K_1: K_1_val, K_2: K_2_val}) - res_desired_val = sess.run(res_desired, {K_1: K_1_val, K_2: K_2_val}) - self.assertAllClose(res_desired_val, res_actual_val) - - - def testHalfKnownRanksTTMatmul(self): - # Tests tt_tt_matmul for the case when one matrice has known ranks - # and the other one doesn't - np.random.seed(1) - K_1 = tf.placeholder(tf.float32, (1, 2, 2, None)) - K_2 = tf.placeholder(tf.float32, (None, 3, 3, 1)) - tt_mat_known_ranks = TensorTrain([K_1, K_2], tt_ranks=[1, 3, 1]) - tt_mat = TensorTrain([K_1, K_2]) - res_actual = ops.full(ops.matmul(tt_mat_known_ranks, tt_mat)) - res_desired = tf.matmul(ops.full(tt_mat_known_ranks), ops.full(tt_mat)) - np.random.seed(1) - K_1_val = np.random.rand(1, 2, 2, 3) - K_2_val = np.random.rand(3, 3, 3, 1) - with self.test_session() as sess: - res_actual_val = sess.run(res_actual, {K_1: K_1_val, K_2: K_2_val}) - res_desired_val = sess.run(res_desired, {K_1: K_1_val, K_2: K_2_val}) - self.assertAllClose(res_desired_val, res_actual_val) - - -class TTTensorBatchTest(tf.test.TestCase): - - def testFullTensor2d(self): - np.random.seed(1) - for rank in [1, 2]: - a = np.random.rand(3, 10, rank).astype(np.float32) - b = np.random.rand(3, rank, 9).astype(np.float32) - tt_cores = (a.reshape(3, 1, 10, rank), b.reshape(3, rank, 9, 1)) - desired = np.einsum('oib,obj->oij', a, b) - with self.test_session(): - tf_tens = TensorTrainBatch(tt_cores) - actual = ops.full(tf_tens) - self.assertAllClose(desired, actual.eval()) - - def testFullTensor3d(self): - np.random.seed(1) - for rank_1 in [1, 2]: - a = np.random.rand(3, 10, rank_1).astype(np.float32) - b = np.random.rand(3, rank_1, 9, 3).astype(np.float32) - c = np.random.rand(3, 3, 8).astype(np.float32) - tt_cores = (a.reshape(3, 1, 10, rank_1), b, c.reshape((3, 3, 8, 1))) - # Basically do full by hand. - desired = np.einsum('oia,oajb,obk->oijk', a, b, c) - with self.test_session(): - tf_tens = TensorTrainBatch(tt_cores) - actual = ops.full(tf_tens) - self.assertAllClose(desired, actual.eval()) - - def testFlatInnerTTTensbyTTTensSameBatchSize(self): - # Inner product between two batch TT-tensors of the same batch_size. - shape_list = ((2, 2), - (2, 3, 4)) - rank_list = (1, 2) - with self.test_session() as sess: - for shape in shape_list: - for rank in rank_list: - tt_1 = initializers.random_tensor_batch(shape, tt_rank=rank, - batch_size=2) - tt_2 = initializers.random_tensor_batch(shape, tt_rank=rank, - batch_size=2) - res_actual = ops.flat_inner(tt_1, tt_2) - tt_1_full = tf.reshape(ops.full(tt_1), (2, 1, -1)) - tt_2_full = tf.reshape(ops.full(tt_2), (2, -1, 1)) - res_desired = tf.matmul(tt_1_full, tt_2_full) - res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) - self.assertAllClose(res_actual_val, np.squeeze(res_desired_val)) - - def testFlatInnerTTTensbyTTTensBroadcasting(self): - # Inner product between two batch TT-tensors with broadcasting. - tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=1) - tt_2 = initializers.random_tensor_batch((2, 3, 4), batch_size=3) - res_actual_1 = ops.flat_inner(tt_1, tt_2) - res_actual_2 = ops.flat_inner(tt_2, tt_1) - res_desired = tf.einsum('ijk,oijk->o', ops.full(tt_1[0]), ops.full(tt_2)) - with self.test_session() as sess: - res = sess.run([res_actual_1, res_actual_2, res_desired]) - res_actual_1_val, res_actual_2_val, res_desired_val = res - self.assertAllClose(res_actual_1_val, res_desired_val) - self.assertAllClose(res_actual_2_val, res_desired_val) - - tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=2) - with self.assertRaises(ValueError): - # The batch_sizes are different. - ops.flat_inner(tt_1, tt_2) - - def testAddSameBatchSize(self): - # Sum two TT-tensors with the same batch size. - tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=3) - tt_b = initializers.random_tensor_batch((2, 1, 4), tt_rank=[1, 2, 4, 1], - batch_size=3) - with self.test_session() as sess: - res_actual = ops.full(ops.add(tt_a, tt_b)) - res_actual2 = ops.full(tt_a + tt_b) - res_desired = ops.full(tt_a) + ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testAddBroadcasting(self): - # Sum two TT-tensors with broadcasting. - tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=1) - tt_b = initializers.random_tensor_batch((2, 1, 4), tt_rank=[1, 2, 4, 1], - batch_size=3) - with self.test_session() as sess: - res_actual = ops.full(ops.add(tt_a, tt_b)) - res_actual2 = ops.full(tt_b + tt_a) - res_desired = ops.full(tt_a) + ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testMultiplyByNumber(self): - # Multiply batch of tensors by a number. - tt = initializers.random_tensor_batch((1, 2, 3), tt_rank=(1, 2, 3, 1), - batch_size=3) - with self.test_session() as sess: - res_actual = ops.full(ops.multiply(tt, 4)) - res_actual2 = ops.full(4.0 * tt) - res_desired = 4.0 * ops.full(tt) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testFrobeniusNormDifferentiableBatch(self): - with self.test_session() as sess: - tt = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) - norm_sq_diff = ops.frobenius_norm_squared(tt, differentiable=True) - variables = [norm_sq_diff, ops.full(tt)] - norm_sq_diff_val, tt_full = sess.run(variables) - desired_norm = np.linalg.norm(tt_full.reshape((5, -1)), axis=1)**2 - self.assertAllClose(norm_sq_diff_val, desired_norm, atol=1e-5, rtol=1e-5) - - def testFrobeniusNormTens(self): - # Frobenius norm of a batch of TT-tensors. - with self.test_session() as sess: - tt = initializers.tensor_batch_with_random_cores((2, 1, 3), batch_size=3) - norm_sq_actual = ops.frobenius_norm_squared(tt) - norm_actual = ops.frobenius_norm(tt) - vars = [norm_sq_actual, norm_actual, ops.full(tt)] - norm_sq_actual_val, norm_actual_val, tt_val = sess.run(vars) - tt_val = tt_val.reshape((3, -1)) - norm_sq_desired_val = np.sum(tt_val * tt_val, axis=1) - norm_desired_val = np.sqrt(norm_sq_desired_val) - self.assertAllClose(norm_sq_actual_val, norm_sq_desired_val) - self.assertAllClose(norm_actual_val, norm_desired_val, atol=1e-5, - rtol=1e-5) - - def testMultiplyBatchByTensor(self): - tt_a = initializers.random_tensor((3, 3, 3), tt_rank=2) - tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) - with self.test_session() as sess: - res_actual = ops.full(ops.multiply(tt_a, tt_b)) - res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) - res_desired = ops.full(tt_a) * ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testMultiplyBatchByBatch(self): - tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) - tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) - res_actual = ops.full(ops.multiply(tt_a, tt_b)) - res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) - res_desired = ops.full(tt_a) * ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - with self.test_session() as sess: - res_actual = ops.full(ops.multiply(tt_a, tt_b)) - res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) - res_desired = ops.full(tt_a) * ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testMultiplyBroadcasting(self): - tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=1) - tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) - with self.test_session() as sess: - res_actual = ops.full(ops.multiply(tt_a, tt_b)) - res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) - res_desired = ops.full(tt_a) * ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testMultiplyUnknownBatchSizeBroadcasting(self): - c1 = tf.placeholder(tf.float32, [None, 1, 3, 2]) - c2 = tf.placeholder(tf.float32, [None, 2, 3, 1]) - tt_a = TensorTrainBatch([c1, c2]) - tt_b = initializers.random_tensor_batch((3, 3), tt_rank=3, batch_size=1) - tt_c = initializers.random_tensor((3, 3), tt_rank=3) - res_ab = ops.full(ops.multiply(tt_a, tt_b)) - res_ba = ops.full(ops.multiply(tt_b, tt_a)) - res_ac = ops.full(ops.multiply(tt_a, tt_c)) - res_ca = ops.full(ops.multiply(tt_c, tt_a)) - res_desired_ab = ops.full(tt_a) * ops.full(tt_b) - res_desired_ac = ops.full(tt_a) * ops.full(tt_c) - to_run = [res_ab, res_ba, res_ac, res_ca, res_desired_ab, res_desired_ac] - feed_dict = {c1:np.random.rand(7, 1, 3, 2), - c2:np.random.rand(7, 2, 3, 1)} - with self.test_session() as sess: - ab, ba, ac, ca, des_ab, des_ac = sess.run(to_run, feed_dict=feed_dict) - self.assertAllClose(ab, des_ab) - self.assertAllClose(ba, des_ab) - self.assertAllClose(ac, des_ac) - self.assertAllClose(ca, des_ac) - - def testMultiplyTwoBatchesUnknownSize(self): - c1 = tf.placeholder(tf.float32, [None, 1, 3, 2]) - c2 = tf.placeholder(tf.float32, [None, 2, 3, 1]) - c3 = tf.placeholder(tf.float32, [None, 1, 3, 2]) - c4 = tf.placeholder(tf.float32, [None, 2, 3, 1]) - tt_a = TensorTrainBatch([c1, c2]) - tt_b = TensorTrainBatch([c3, c4]) - res_ab = ops.full(ops.multiply(tt_a, tt_b)) - res_ba = ops.full(ops.multiply(tt_b, tt_a)) - res_desired = ops.full(tt_a) * ops.full(tt_b) - to_run = [res_ab, res_ba, res_desired] - feed_dict = {c1:np.random.rand(7, 1, 3, 2), - c2:np.random.rand(7, 2, 3, 1), - c3:np.random.rand(7, 1, 3, 2), - c4:np.random.rand(7, 2, 3, 1)} - - feed_dict_err = {c1:np.random.rand(7, 1, 3, 2), - c2:np.random.rand(7, 2, 3, 1), - c3:np.random.rand(1, 1, 3, 2), - c4:np.random.rand(1, 2, 3, 1)} - - with self.test_session() as sess: - ab_full, ba_full, des_full = sess.run(to_run, feed_dict=feed_dict) - self.assertAllClose(ab_full, des_full) - self.assertAllClose(ba_full, des_full) - with self.assertRaises(tf.errors.InvalidArgumentError): - sess.run(to_run, feed_dict=feed_dict_err) - - def testMultiplyUnknownSizeBatchAndBatch(self): - c1 = tf.placeholder(tf.float32, [None, 1, 3, 2]) - c2 = tf.placeholder(tf.float32, [None, 2, 3, 1]) - tt_b = initializers.random_tensor_batch((3, 3), tt_rank=2, batch_size=8) - tt_a = TensorTrainBatch([c1, c2]) - res_ab = ops.full(ops.multiply(tt_a, tt_b)) - res_ba = ops.full(ops.multiply(tt_b, tt_a)) - res_desired = ops.full(tt_a) * ops.full(tt_b) - to_run = [res_ab, res_ba, res_desired] - feed_dict = {c1:np.random.rand(8, 1, 3, 2), - c2:np.random.rand(8, 2, 3, 1)} - - feed_dict_err = {c1:np.random.rand(1, 1, 3, 2), - c2:np.random.rand(1, 2, 3, 1)} - - with self.test_session() as sess: - ab_full, ba_full, des_full = sess.run(to_run, feed_dict=feed_dict) - self.assertAllClose(ab_full, des_full) - self.assertAllClose(ba_full, des_full) - with self.assertRaises(tf.errors.InvalidArgumentError): - sess.run(to_run, feed_dict=feed_dict_err) - - def testGatherND(self): - idx = [[0, 0, 0], [0, 1, 2], [0, 1, 0]] - pl_idx = tf.placeholder(tf.int32, [None, 3]) - tt = initializers.random_tensor((3, 4, 5), tt_rank=2) - res_np = ops.gather_nd(tt, idx) - res_pl = ops.gather_nd(tt, pl_idx) - res_desired = tf.gather_nd(ops.full(tt), idx) - to_run = [res_np, res_pl, res_desired] - with self.test_session() as sess: - res_np_v, res_pl_v, des_v = sess.run(to_run, feed_dict={pl_idx: idx}) - self.assertAllClose(res_np_v, des_v) - self.assertAllClose(res_pl_v, res_pl_v) - - def testGatherNDBatch(self): - idx = [[0, 0, 0, 0], [1, 0, 1, 2], [0, 0, 1, 0]] - pl_idx = tf.placeholder(tf.int32, [None, 4]) - tt = initializers.random_tensor_batch((3, 4, 5), tt_rank=2, batch_size=2) - res_np = ops.gather_nd(tt, idx) - res_pl = ops.gather_nd(tt, pl_idx) - res_desired = tf.gather_nd(ops.full(tt), idx) - to_run = [res_np, res_pl, res_desired] - with self.test_session() as sess: - res_np_v, res_pl_v, des_v = sess.run(to_run, feed_dict={pl_idx: idx}) - self.assertAllClose(res_np_v, des_v) - self.assertAllClose(res_pl_v, res_pl_v) - - def testCoreRenormBatch(self): - a = initializers.random_tensor_batch(3 * (10,), tt_rank=7, batch_size=5) - b = ops.renormalize_tt_cores(a) - var_list = [ops.full(a), ops.full(b)] - - with self.test_session() as sess: - af, bf = sess.run(var_list) - b_cores = sess.run(b.tt_cores) - b_cores_norms = [] - for cr in b_cores: - b_cores_norms.append(np.linalg.norm(cr)) - self.assertAllClose(af, bf, atol=1e-5, rtol=1e-5) - self.assertAllClose(b_cores_norms, b_cores_norms[0] - * np.ones((len(b_cores)))) - -class TTMatrixTestBatch(tf.test.TestCase): - - def testFullMatrix2d(self): - np.random.seed(1) - for rank in [1, 2]: - a = np.random.rand(3, 2, 3, rank).astype(np.float32) - b = np.random.rand(3, rank, 4, 5).astype(np.float32) - tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape((3, rank, 4, 5, 1))) - # Basically do full by hand. - desired = np.einsum('oijb,obkl->oijkl', a, b) - desired = desired.reshape((3, 2, 3, 4, 5)) - desired = desired.transpose((0, 1, 3, 2, 4)) - desired = desired.reshape((3, 2 * 4, 3 * 5)) - with self.test_session(): - tf_mat = TensorTrainBatch(tt_cores) - actual = ops.full(tf_mat) - self.assertAllClose(desired, actual.eval()) - - def testFullMatrix3d(self): - np.random.seed(1) - for rank in [1, 2]: - a = np.random.rand(3, 2, 3, rank).astype(np.float32) - b = np.random.rand(3, rank, 4, 5, rank).astype(np.float32) - c = np.random.rand(3, rank, 2, 2).astype(np.float32) - tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape(3, rank, 4, 5, rank), - c.reshape(3, rank, 2, 2, 1)) - # Basically do full by hand. - desired = np.einsum('oija,oaklb,obpq->oijklpq', a, b, c) - desired = desired.reshape((3, 2, 3, 4, 5, 2, 2)) - desired = desired.transpose((0, 1, 3, 5, 2, 4, 6)) - desired = desired.reshape((3, 2 * 4 * 2, 3 * 5 * 2)) - with self.test_session(): - tf_mat = TensorTrainBatch(tt_cores) - actual = ops.full(tf_mat) - self.assertAllClose(desired, actual.eval()) - - def testTTMatTimesTTMatSameBatchSize(self): - # Multiply a batch of TT-matrices by another batch of TT-matrices with the - # same batch sizes. - left_shape = (2, 3) - sum_shape = (4, 3) - right_shape = (4, 4) - with self.test_session() as sess: - tt_mat_1 = initializers.random_matrix_batch((left_shape, sum_shape), - tt_rank=3, batch_size=3) - tt_mat_2 = initializers.random_matrix_batch((sum_shape, right_shape), - batch_size=3) - res_actual = ops.matmul(tt_mat_1, tt_mat_2) - res_actual = ops.full(res_actual) - res_desired = tf.matmul(ops.full(tt_mat_1), ops.full(tt_mat_2)) - res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) - # TODO: why so bad accuracy? - self.assertAllClose(res_actual_val, res_desired_val, atol=1e-5, rtol=1e-5) - - def testTTMatTimesTTMatBroadcasting(self): - # Multiply a batch of TT-matrices by another batch of TT-matrices with - # broadcasting. - left_shape = (2, 3) - sum_shape = (4, 3) - right_shape = (4, 4) - with self.test_session() as sess: - tt_mat_1 = initializers.random_matrix_batch((left_shape, sum_shape), - tt_rank=3, batch_size=3) - tt_mat_2 = initializers.random_matrix_batch((sum_shape, right_shape)) - # TT-batch by one element TT-batch - res_actual = ops.matmul(tt_mat_1, tt_mat_2) - res_actual = ops.full(res_actual) - # TT by TT-batch. - res_actual2 = ops.matmul(ops.transpose(tt_mat_2[0]), ops.transpose(tt_mat_1)) - res_actual2 = ops.full(ops.transpose(res_actual2)) - res_desired = tf.einsum('oij,jk->oik', ops.full(tt_mat_1), - ops.full(tt_mat_2[0])) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val, atol=1e-5, rtol=1e-5) - self.assertAllClose(res_actual2_val, res_desired_val, atol=1e-5, - rtol=1e-5) - - def testTranspose(self): - # Transpose a batch of TT-matrices. - with self.test_session() as sess: - tt = initializers.random_matrix_batch(((2, 3, 4), (2, 2, 2)), batch_size=2) - res_actual = ops.full(ops.transpose(tt)) - res_actual_val, tt_val = sess.run([res_actual, ops.full(tt)]) - self.assertAllClose(tt_val.transpose((0, 2, 1)), res_actual_val) - - def testAddSameBatchSize(self): - # Sum two TT-matrices with the same batch size. - tt_a = initializers.random_matrix_batch(((2, 1, 4), None), tt_rank=2, - batch_size=3) - tt_b = initializers.random_matrix_batch(((2, 1, 4), None), - tt_rank=[1, 2, 4, 1], batch_size=3) - with self.test_session() as sess: - res_actual = ops.full(ops.add(tt_a, tt_b)) - res_actual2 = ops.full(tt_a + tt_b) - res_desired = ops.full(tt_a) + ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testAddBroadcasting(self): - # Sum two TT-matrices with broadcasting. - tt_a = initializers.random_matrix_batch(((2, 1, 4), (2, 2, 2)), tt_rank=2, - batch_size=3) - tt_b = initializers.random_matrix_batch(((2, 1, 4), (2, 2, 2)), - tt_rank=[1, 2, 4, 1], batch_size=1) - with self.test_session() as sess: - res_actual = ops.full(ops.add(tt_a, tt_b)) - res_actual2 = ops.full(tt_b + tt_a) - res_desired = ops.full(tt_a) + ops.full(tt_b) - to_run = [res_actual, res_actual2, res_desired] - res_actual_val, res_actual2_val, res_desired_val = sess.run(to_run) - self.assertAllClose(res_actual_val, res_desired_val) - self.assertAllClose(res_actual2_val, res_desired_val) - - def testCastFloat(self): - # Test cast function for float tt-matrices and vectors. - tt_mat = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=2, - batch_size=3) - - with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_mat, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) - - def testCastIntFloat(self): - # Tests cast function from int to float for matrices. - np.random.seed(1) - K_1 = np.random.randint(0, high=100, size=(1, 2, 2, 2)) - K_2 = np.random.randint(0, high=100, size=(2, 3, 3, 2)) - K_3 = np.random.randint(0, high=100, size=(2, 2, 2, 1)) - tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) - tt_int_batch = shapes.expand_batch_dim(tt_int) - - with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_int_batch, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) - - -def _random_sparse(shape, non_zeros): - sparse_flat_indices = np.random.choice(np.prod(shape), non_zeros).astype(int) - sparse_indices = np.unravel_index(sparse_flat_indices, shape) - sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(non_zeros).astype(np.float32) - sparse = tf.SparseTensor(indices=sparse_indices, values=values, - dense_shape=shape) - return sparse - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/regularizers.py b/build/lib/t3f/regularizers.py deleted file mode 100644 index 6928e329..00000000 --- a/build/lib/t3f/regularizers.py +++ /dev/null @@ -1,79 +0,0 @@ -import numbers - -import tensorflow as tf - -from t3f import ops - - -def l2_regularizer(scale, scope=None): - """Returns a function that applies L2 regularization to TensorTrain weights. - - Args: - scale: A scalar multiplier `Tensor`. 0.0 disables the regularizer. - scope: An optional scope name. - - Returns: - A function with signature `l2(tt)` that applies L2 regularization. - - Raises: - ValueError: If scale is negative or if scale is not a float. - """ - if isinstance(scale, numbers.Integral): - raise ValueError('scale cannot be an integer: %s' % (scale,)) - if isinstance(scale, numbers.Real): - if scale < 0.: - raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % - scale) - if scale == 0.: - tf.logging.info('Scale of 0 disables regularizer.') - return lambda _: None - - def l2(tt): - """Applies l2 regularization to TensorTrain object.""" - with tf.name_scope(scope, 'l2_regularizer', [tt]) as name: - my_scale = tf.convert_to_tensor(scale, - dtype=tt.dtype.base_dtype, - name='scale') - return tf.mul(my_scale, ops.frobenius_norm_squared(tt), name=name) - - return l2 - - -def cores_regularizer(core_regularizer, scale, scope=None): - """Returns a function that applies given regularization to each TT-core. - - Args: - core_regularizer: a function with signature `core_regularizer(core)` that - returns the penalty for the given TT-core. - scale: A scalar multiplier `Tensor`. 0.0 disables the regularizer. - scope: An optional scope name. - - Returns: - A function with signature `regularizer(weights)` that applies - the regularization. - - Raises: - ValueError: If scale is negative or if scale is not a float. - """ - if isinstance(scale, numbers.Integral): - raise ValueError('scale cannot be an integer: %s' % (scale,)) - if isinstance(scale, numbers.Real): - if scale < 0.: - raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % - scale) - if scale == 0.: - tf.logging.info('Scale of 0 disables regularizer.') - return lambda _: None - - def regularizer(tt): - """Applies the regularization to TensorTrain object.""" - with tf.name_scope(scope, 'l2_regularizer', [tt]) as name: - my_scale = tf.convert_to_tensor(scale, - dtype=tt.dtype.base_dtype, - name='scale') - penalty = 0.0 - for i in range(tt.ndims()): - penalty += core_regularizer(tt.tt_cores[i]) - return tf.mul(my_scale, penalty, name=name) - - return regularizer diff --git a/build/lib/t3f/riemannian.py b/build/lib/t3f/riemannian.py deleted file mode 100644 index 21a49748..00000000 --- a/build/lib/t3f/riemannian.py +++ /dev/null @@ -1,751 +0,0 @@ -import tensorflow as tf - -from t3f.tensor_train import TensorTrain -from t3f.tensor_train_batch import TensorTrainBatch -from t3f import shapes -from t3f import decompositions - - -def project_sum(what, where, weights=None): - """Project sum of `what` TTs on the tangent space of `where` TT. - - project_sum(what, x) = P_x(what) - project_sum(batch_what, x) = P_x(\sum_i batch_what[i]) - project_sum(batch_what, x, weights) = P_x(\sum_j weights[j] * batch_what[j]) - - This function implements the algorithm from the paper [1], theorem 3.1. - - [1] C. Lubich, I. Oseledets and B. Vandereycken, Time integration of - Tensor Trains. - - Args: - what: TensorTrain or TensorTrainBatch. In the case of batch returns - projection of the sum of elements in the batch. - where: TensorTrain, TT-tensor or TT-matrix on which tangent space to project - weights: python list or tf.Tensor of numbers or None, weights of the sum - - Returns: - a TensorTrain with the TT-ranks equal 2 * tangent_space_tens.get_tt_ranks() - - Complexity: - O(d r_where^3 m) for orthogonalizing the TT-cores of where - +O(batch_size d r_what r_where n (r_what + r_where)) - d is the number of TT-cores (what.ndims()); - r_what is the largest TT-rank of what max(what.get_tt_rank()) - r_where is the largest TT-rank of where - n is the size of the axis dimension of what and where e.g. - for a tensor of size 4 x 4 x 4, n is 4; - for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 - """ - # Always work with batch of TT objects for simplicity. - what = shapes.expand_batch_dim(what) - - if weights is not None: - weights = tf.convert_to_tensor(weights) - - if not isinstance(where, TensorTrain): - raise ValueError('The first argument should be a TensorTrain object, got ' - '"%s".' % where) - - if where.get_raw_shape() != what.get_raw_shape(): - raise ValueError('The shapes of the tensor we want to project and of the ' - 'tensor on which tangent space we want to project should ' - 'match, got %s and %s.' % - (where.get_raw_shape(), - what.get_raw_shape())) - - dtypes_compatible = (where.dtype.is_compatible_with(what.dtype) or - what.dtype.is_compatible_with(where.dtype)) - if not dtypes_compatible: - raise ValueError('Dtypes of the arguments should coincide, got %s and %s.' % - (where.dtype, - what.dtype)) - - left_tangent_space_tens = decompositions.orthogonalize_tt_cores( - where) - right_tangent_space_tens = decompositions.orthogonalize_tt_cores( - left_tangent_space_tens, left_to_right=False) - - ndims = where.ndims() - dtype = where.dtype - raw_shape = shapes.lazy_raw_shape(where) - batch_size = shapes.lazy_batch_size(what) - right_tangent_tt_ranks = shapes.lazy_tt_ranks(right_tangent_space_tens) - left_tangent_tt_ranks = shapes.lazy_tt_ranks(left_tangent_space_tens) - - # For einsum notation. - mode_str = 'ij' if where.is_tt_matrix() else 'i' - right_rank_dim = where.right_tt_rank_dim - left_rank_dim = where.left_tt_rank_dim - if weights is not None: - weights_shape = weights.get_shape() - output_is_batch = len(weights_shape) > 1 and weights_shape[1] > 1 - else: - output_is_batch = False - output_batch_str = 'o' if output_is_batch else '' - if output_is_batch: - right_rank_dim += 1 - left_rank_dim += 1 - output_batch_size = weights.get_shape()[1].value - - # Prepare rhs vectors. - # rhs[core_idx] is of size - # batch_size x tensor_tt_ranks[core_idx] x tangent_tt_ranks[core_idx] - rhs = [None] * (ndims + 1) - rhs[ndims] = tf.ones((batch_size, 1, 1), dtype=dtype) - for core_idx in range(ndims - 1, 0, -1): - tens_core = what.tt_cores[core_idx] - right_tang_core = right_tangent_space_tens.tt_cores[core_idx] - einsum_str = 'sa{0}b,sbd,c{0}d->sac'.format(mode_str) - rhs[core_idx] = tf.einsum(einsum_str, tens_core, rhs[core_idx + 1], - right_tang_core) - - # Prepare lhs vectors. - # lhs[core_idx] is of size - # batch_size x tangent_tt_ranks[core_idx] x tensor_tt_ranks[core_idx] - lhs = [None] * (ndims + 1) - lhs[0] = tf.ones((batch_size, 1, 1), dtype=dtype) - for core_idx in range(ndims - 1): - tens_core = what.tt_cores[core_idx] - left_tang_core = left_tangent_space_tens.tt_cores[core_idx] - einsum_str = 'sab,a{0}c,sb{0}d->scd'.format(mode_str) - lhs[core_idx + 1] = tf.einsum(einsum_str, lhs[core_idx], left_tang_core, - tens_core) - - # Left to right sweep. - res_cores_list = [] - for core_idx in range(ndims): - tens_core = what.tt_cores[core_idx] - left_tang_core = left_tangent_space_tens.tt_cores[core_idx] - right_tang_core = right_tangent_space_tens.tt_cores[core_idx] - - if core_idx < ndims - 1: - einsum_str = 'sab,sb{0}c->sa{0}c'.format(mode_str) - proj_core = tf.einsum(einsum_str, lhs[core_idx], tens_core) - einsum_str = 'a{0}b,sbc->sa{0}c'.format(mode_str) - proj_core -= tf.einsum(einsum_str, left_tang_core, lhs[core_idx + 1]) - if weights is None: - einsum_str = 'sa{0}b,sbc->a{0}c'.format(mode_str) - proj_core = tf.einsum(einsum_str, proj_core, rhs[core_idx + 1]) - else: - einsum_str = 'sa{0}b,sbc->sa{0}c'.format(mode_str, output_batch_str) - proj_core_s = tf.einsum(einsum_str, proj_core, rhs[core_idx + 1]) - einsum_str = 's{1},sa{0}c->{1}a{0}c'.format(mode_str, output_batch_str) - proj_core = tf.einsum(einsum_str, weights, proj_core_s) - - if core_idx == ndims - 1: - if weights is None: - einsum_str = 'sab,sb{0}c->a{0}c'.format(mode_str) - proj_core = tf.einsum(einsum_str, lhs[core_idx], tens_core) - else: - einsum_str = 'sab,sb{0}c->sa{0}c'.format(mode_str, output_batch_str) - proj_core_s = tf.einsum(einsum_str, lhs[core_idx], tens_core) - einsum_str = 's{1},sa{0}c->{1}a{0}c'.format(mode_str, output_batch_str) - proj_core = tf.einsum(einsum_str, weights, proj_core_s) - - if output_is_batch: - # Add batch dimension of size output_batch_size to left_tang_core and - # right_tang_core - extended_left_tang_core = tf.expand_dims(left_tang_core, 0) - extended_right_tang_core = tf.expand_dims(right_tang_core, 0) - if where.is_tt_matrix(): - extended_left_tang_core = tf.tile(extended_left_tang_core, - [output_batch_size, 1, 1, 1, 1]) - extended_right_tang_core = tf.tile(extended_right_tang_core, - [output_batch_size, 1, 1, 1, 1]) - else: - extended_left_tang_core = tf.tile(extended_left_tang_core, - [output_batch_size, 1, 1, 1]) - extended_right_tang_core = tf.tile(extended_right_tang_core, - [output_batch_size, 1, 1, 1]) - else: - extended_left_tang_core = left_tang_core - extended_right_tang_core = right_tang_core - - if core_idx == 0: - res_core = tf.concat((proj_core, extended_left_tang_core), - axis=right_rank_dim) - elif core_idx == ndims - 1: - res_core = tf.concat((extended_right_tang_core, proj_core), axis=left_rank_dim) - else: - rank_1 = right_tangent_tt_ranks[core_idx] - rank_2 = left_tangent_tt_ranks[core_idx + 1] - if where.is_tt_matrix(): - mode_size_n = raw_shape[0][core_idx] - mode_size_m = raw_shape[1][core_idx] - shape = [rank_1, mode_size_n, mode_size_m, rank_2] - else: - mode_size = raw_shape[0][core_idx] - shape = [rank_1, mode_size, rank_2] - if output_is_batch: - shape = [output_batch_size] + shape - zeros = tf.zeros(shape, dtype) - upper = tf.concat((extended_right_tang_core, zeros), axis=right_rank_dim) - lower = tf.concat((proj_core, extended_left_tang_core), - axis=right_rank_dim) - res_core = tf.concat((upper, lower), axis=left_rank_dim) - res_cores_list.append(res_core) - # TODO: TT-ranks. - if output_is_batch: - res = TensorTrainBatch(res_cores_list, where.get_raw_shape(), - batch_size=output_batch_size) - else: - res = TensorTrain(res_cores_list, where.get_raw_shape()) - - res.projection_on = where - return res - - -def project(what, where): - """Project `what` TTs on the tangent space of `where` TT. - - project(what, x) = P_x(what) - project(batch_what, x) = batch(P_x(batch_what[0]), ..., P_x(batch_what[N])) - - This function implements the algorithm from the paper [1], theorem 3.1. - - [1] C. Lubich, I. Oseledets and B. Vandereycken, Time integration of - Tensor Trains. - - Args: - what: TensorTrain or TensorTrainBatch. In the case of batch returns - batch with projection of each individual tensor. - where: TensorTrain, TT-tensor or TT-matrix on which tangent space to project - - Returns: - a TensorTrain with the TT-ranks equal 2 * tangent_space_tens.get_tt_ranks() - - Complexity: - O(d r_where^3 m) for orthogonalizing the TT-cores of where - +O(batch_size d r_what r_where n (r_what + r_where)) - d is the number of TT-cores (what.ndims()); - r_what is the largest TT-rank of what max(what.get_tt_rank()) - r_where is the largest TT-rank of where - n is the size of the axis dimension of what and where e.g. - for a tensor of size 4 x 4 x 4, n is 4; - for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12 - """ - - if not isinstance(where, TensorTrain): - raise ValueError('The first argument should be a TensorTrain object, got ' - '"%s".' % where) - - if where.get_raw_shape() != what.get_raw_shape(): - raise ValueError('The shapes of the tensor we want to project and of the ' - 'tensor on which tangent space we want to project should ' - 'match, got %s and %s.' % - (where.get_raw_shape(), - what.get_raw_shape())) - dtypes_compatible = (where.dtype.is_compatible_with(what.dtype) or - what.dtype.is_compatible_with(where.dtype)) - if not dtypes_compatible: - raise ValueError('Dtypes of the arguments should coincide, got %s and %s.' % - (where.dtype, - what.dtype)) - - left_tangent_space_tens = decompositions.orthogonalize_tt_cores( - where) - right_tangent_space_tens = decompositions.orthogonalize_tt_cores( - left_tangent_space_tens, left_to_right=False) - - ndims = where.ndims() - dtype = where.dtype - raw_shape = shapes.lazy_raw_shape(where) - right_tangent_tt_ranks = shapes.lazy_tt_ranks(right_tangent_space_tens) - left_tangent_tt_ranks = shapes.lazy_tt_ranks(left_tangent_space_tens) - - # For einsum notation. - mode_str = 'ij' if where.is_tt_matrix() else 'i' - right_rank_dim = what.right_tt_rank_dim - left_rank_dim = what.left_tt_rank_dim - output_is_batch = isinstance(what, TensorTrainBatch) - if output_is_batch: - output_batch_size = what.batch_size - - # Always work with batch of TT objects for simplicity. - what = shapes.expand_batch_dim(what) - batch_size = shapes.lazy_batch_size(what) - - # Prepare rhs vectors. - # rhs[core_idx] is of size - # batch_size x tensor_tt_ranks[core_idx] x tangent_tt_ranks[core_idx] - rhs = [None] * (ndims + 1) - rhs[ndims] = tf.ones((batch_size, 1, 1), dtype=dtype) - for core_idx in range(ndims - 1, 0, -1): - tens_core = what.tt_cores[core_idx] - right_tang_core = right_tangent_space_tens.tt_cores[core_idx] - einsum_str = 'sa{0}b,sbd,c{0}d->sac'.format(mode_str) - rhs[core_idx] = tf.einsum(einsum_str, tens_core, rhs[core_idx + 1], - right_tang_core) - - # Prepare lhs vectors. - # lhs[core_idx] is of size - # batch_size x tangent_tt_ranks[core_idx] x tensor_tt_ranks[core_idx] - lhs = [None] * (ndims + 1) - lhs[0] = tf.ones((batch_size, 1, 1), dtype=dtype) - for core_idx in range(ndims - 1): - tens_core = what.tt_cores[core_idx] - left_tang_core = left_tangent_space_tens.tt_cores[core_idx] - einsum_str = 'sab,a{0}c,sb{0}d->scd'.format(mode_str) - lhs[core_idx + 1] = tf.einsum(einsum_str, lhs[core_idx], left_tang_core, - tens_core) - - # Left to right sweep. - res_cores_list = [] - for core_idx in range(ndims): - tens_core = what.tt_cores[core_idx] - left_tang_core = left_tangent_space_tens.tt_cores[core_idx] - right_tang_core = right_tangent_space_tens.tt_cores[core_idx] - - if core_idx < ndims - 1: - einsum_str = 'sab,sb{0}c->sa{0}c'.format(mode_str) - proj_core = tf.einsum(einsum_str, lhs[core_idx], tens_core) - einsum_str = 'a{0}b,sbc->sa{0}c'.format(mode_str) - proj_core -= tf.einsum(einsum_str, left_tang_core, lhs[core_idx + 1]) - if output_is_batch: - einsum_str = 'sa{0}b,sbc->sa{0}c'.format(mode_str) - else: - einsum_str = 'sa{0}b,sbc->a{0}c'.format(mode_str) - proj_core = tf.einsum(einsum_str, proj_core, rhs[core_idx + 1]) - - if core_idx == ndims - 1: - if output_is_batch: - einsum_str = 'sab,sb{0}c->sa{0}c'.format(mode_str) - else: - einsum_str = 'sab,sb{0}c->a{0}c'.format(mode_str) - proj_core = tf.einsum(einsum_str, lhs[core_idx], tens_core) - - if output_is_batch: - # Add batch dimension of size output_batch_size to left_tang_core and - # right_tang_core - extended_left_tang_core = tf.expand_dims(left_tang_core, 0) - extended_right_tang_core = tf.expand_dims(right_tang_core, 0) - if where.is_tt_matrix(): - extended_left_tang_core = tf.tile(extended_left_tang_core, - [output_batch_size, 1, 1, 1, 1]) - extended_right_tang_core = tf.tile(extended_right_tang_core, - [output_batch_size, 1, 1, 1, 1]) - else: - extended_left_tang_core = tf.tile(extended_left_tang_core, - [output_batch_size, 1, 1, 1]) - extended_right_tang_core = tf.tile(extended_right_tang_core, - [output_batch_size, 1, 1, 1]) - else: - extended_left_tang_core = left_tang_core - extended_right_tang_core = right_tang_core - - if core_idx == 0: - res_core = tf.concat((proj_core, extended_left_tang_core), - axis=right_rank_dim) - elif core_idx == ndims - 1: - res_core = tf.concat((extended_right_tang_core, proj_core), axis=left_rank_dim) - else: - rank_1 = right_tangent_tt_ranks[core_idx] - rank_2 = left_tangent_tt_ranks[core_idx + 1] - if where.is_tt_matrix(): - mode_size_n = raw_shape[0][core_idx] - mode_size_m = raw_shape[1][core_idx] - shape = [rank_1, mode_size_n, mode_size_m, rank_2] - else: - mode_size = raw_shape[0][core_idx] - shape = [rank_1, mode_size, rank_2] - if output_is_batch: - shape = [output_batch_size] + shape - zeros = tf.zeros(shape, dtype) - upper = tf.concat((extended_right_tang_core, zeros), axis=right_rank_dim) - lower = tf.concat((proj_core, extended_left_tang_core), - axis=right_rank_dim) - res_core = tf.concat((upper, lower), axis=left_rank_dim) - res_cores_list.append(res_core) - # TODO: TT-ranks. - if output_is_batch: - res = TensorTrainBatch(res_cores_list, where.get_raw_shape(), - batch_size=output_batch_size) - else: - res = TensorTrain(res_cores_list, where.get_raw_shape()) - - res.projection_on = where - return res - - -def project_matmul(what, where, matrix): - """Project `matrix` * `what` TTs on the tangent space of `where` TT. - - project(what, x) = P_x(what) - project(batch_what, x) = batch(P_x(batch_what[0]), ..., P_x(batch_what[N])) - - This function implements the algorithm from the paper [1], theorem 3.1. - - [1] C. Lubich, I. Oseledets and B. Vandereycken, Time integration of - Tensor Trains. - - Args: - what: TensorTrain or TensorTrainBatch. In the case of batch returns - batch with projection of each individual tensor. - where: TensorTrain, TT-tensor or TT-matrix on which tangent space to project - matrix: TensorTrain, TT-matrix to multiply by what - - Returns: - a TensorTrain with the TT-ranks equal 2 * tangent_space_tens.get_tt_ranks() - - Complexity: - O(d r_where^3 m) for orthogonalizing the TT-cores of where - +O(batch_size d R r_what r_where (n r_what + n m R + m r_where)) - d is the number of TT-cores (what.ndims()); - r_what is the largest TT-rank of what max(what.get_tt_rank()) - r_where is the largest TT-rank of where - matrix is of TT-rank R and of raw-shape (m, m, ..., m) x (n, n, ..., n). - """ - - if not isinstance(where, TensorTrain): - raise ValueError('The first argument should be a TensorTrain object, got ' - '"%s".' % where) - - if where.get_raw_shape() != what.get_raw_shape(): - raise ValueError('The shapes of the tensor we want to project and of the ' - 'tensor on which tangent space we want to project should ' - 'match, got %s and %s.' % - (where.get_raw_shape(), - what.get_raw_shape())) - - dtypes_compatible = (where.dtype.is_compatible_with(what.dtype) or - what.dtype.is_compatible_with(where.dtype)) - if not dtypes_compatible: - raise ValueError('Dtypes of the arguments should coincide, got %s and %s.' % - (where.dtype, - what.dtype)) - - left_tangent_space_tens = decompositions.orthogonalize_tt_cores( - where) - right_tangent_space_tens = decompositions.orthogonalize_tt_cores( - left_tangent_space_tens, left_to_right=False) - - ndims = where.ndims() - dtype = where.dtype - raw_shape = shapes.lazy_raw_shape(where) - batch_size = shapes.lazy_batch_size(what) - right_tangent_tt_ranks = shapes.lazy_tt_ranks(right_tangent_space_tens) - left_tangent_tt_ranks = shapes.lazy_tt_ranks(left_tangent_space_tens) - - # For einsum notation. - right_rank_dim = what.right_tt_rank_dim - left_rank_dim = what.left_tt_rank_dim - output_is_batch = isinstance(what, TensorTrainBatch) - if output_is_batch: - output_batch_size = what.batch_size - - # Always work with batch of TT objects for simplicity. - what = shapes.expand_batch_dim(what) - - # Prepare rhs vectors. - # rhs[core_idx] is of size - # batch_size x tensor_tt_ranks[core_idx] x matrix_tt_ranks[core_idx] x tangent_tt_ranks[core_idx] - rhs = [None] * (ndims + 1) - rhs[ndims] = tf.ones((batch_size, 1, 1, 1), dtype=dtype) - for core_idx in range(ndims - 1, 0, -1): - tens_core = what.tt_cores[core_idx] - right_tang_core = right_tangent_space_tens.tt_cores[core_idx] - matrix_core = matrix.tt_cores[core_idx] - rhs[core_idx] = tf.einsum('bije,cikf,sdef,sajkd->sabc', matrix_core, - right_tang_core, rhs[core_idx + 1], tens_core) - # Prepare lhs vectors. - # lhs[core_idx] is of size - # batch_size x tangent_tt_ranks[core_idx] x matrix_tt_ranks[core_idx] x tensor_tt_ranks[core_idx] - lhs = [None] * (ndims + 1) - lhs[0] = tf.ones((batch_size, 1, 1, 1), dtype=dtype) - for core_idx in range(ndims - 1): - tens_core = what.tt_cores[core_idx] - left_tang_core = left_tangent_space_tens.tt_cores[core_idx] - matrix_core = matrix.tt_cores[core_idx] - # TODO: brutforce order of indices in lhs?? - lhs[core_idx + 1] = tf.einsum('bije,aikd,sabc,scjkf->sdef', matrix_core, - left_tang_core, lhs[core_idx], tens_core) - - # Left to right sweep. - res_cores_list = [] - for core_idx in range(ndims): - tens_core = what.tt_cores[core_idx] - matrix_core = matrix.tt_cores[core_idx] - left_tang_core = left_tangent_space_tens.tt_cores[core_idx] - right_tang_core = right_tangent_space_tens.tt_cores[core_idx] - - if core_idx < ndims - 1: - proj_core = tf.einsum('scjke,sabc,bijd->saikde', tens_core, - lhs[core_idx], matrix_core) - proj_core -= tf.einsum('aikb,sbcd->saikcd', left_tang_core, - lhs[core_idx + 1]) - proj_core = tf.einsum('saikcb,sbcd->saikd', proj_core, rhs[core_idx + 1]) - - if core_idx == ndims - 1: - # d and e dimensions take 1 value, since its the last rank. - # To make the result shape (?, ?, ?, 1), we are summing d and leaving e, - # but we could have done the opposite -- sum e and leave d. - proj_core = tf.einsum('sabc,bijd,scjke->saike', lhs[core_idx], matrix_core, - tens_core) - - if output_is_batch: - # Add batch dimension of size output_batch_size to left_tang_core and - # right_tang_core - extended_left_tang_core = tf.expand_dims(left_tang_core, 0) - extended_right_tang_core = tf.expand_dims(right_tang_core, 0) - extended_left_tang_core = tf.tile(extended_left_tang_core, - [output_batch_size, 1, 1, 1, 1]) - extended_right_tang_core = tf.tile(extended_right_tang_core, - [output_batch_size, 1, 1, 1, 1]) - else: - extended_left_tang_core = left_tang_core - extended_right_tang_core = right_tang_core - - if core_idx == 0: - res_core = tf.concat((proj_core, extended_left_tang_core), - axis=right_rank_dim) - elif core_idx == ndims - 1: - res_core = tf.concat((extended_right_tang_core, proj_core), - axis=left_rank_dim) - else: - rank_1 = right_tangent_tt_ranks[core_idx] - rank_2 = left_tangent_tt_ranks[core_idx + 1] - mode_size_n = raw_shape[0][core_idx] - mode_size_m = raw_shape[1][core_idx] - shape = [rank_1, mode_size_n, mode_size_m, rank_2] - if output_is_batch: - shape = [output_batch_size] + shape - zeros = tf.zeros(shape, dtype) - upper = tf.concat((extended_right_tang_core, zeros), - axis=right_rank_dim) - lower = tf.concat((proj_core, extended_left_tang_core), - axis=right_rank_dim) - res_core = tf.concat((upper, lower), axis=left_rank_dim) - res_cores_list.append(res_core) - - # TODO: TT-ranks. - if output_is_batch: - res = TensorTrainBatch(res_cores_list, where.get_raw_shape(), - batch_size=output_batch_size) - else: - res = TensorTrain(res_cores_list, where.get_raw_shape()) - - res.projection_on = where - return res - - -def pairwise_flat_inner_projected(projected_tt_vectors_1, - projected_tt_vectors_2): - """Scalar products between two batches of TTs from the same tangent space. - - res[i, j] = t3f.flat_inner(projected_tt_vectors_1[i], projected_tt_vectors_1[j]). - - pairwise_flat_inner_projected(projected_tt_vectors_1, projected_tt_vectors_2) - is equivalent to - pairwise_flat_inner(projected_tt_vectors_1, projected_tt_vectors_2) - , but works only on objects from the same tangent space and is much faster - than general pairwise_flat_inner. - - Args: - projected_tt_vectors_1: TensorTrainBatch of tensors projected on the same - tangent space as projected_tt_vectors_2. - projected_tt_vectors_2: TensorTrainBatch. - - Returns: - tf.tensor with the scalar product matrix. - - Complexity: - O(batch_size^2 d r^2 n), where - d is the number of TT-cores (projected_tt_vectors_1.ndims()); - r is the largest TT-rank max(projected_tt_vectors_1.get_tt_rank()) - (i.e. 2 * {the TT-rank of the object we projected vectors onto}. - and n is the size of the axis dimension, e.g. - for a tensor of size 4 x 4 x 4, n is 4; - for a 9 x 64 matrix of raw shape (3, 3, 3) x (4, 4, 4) n is 12. - """ - if not hasattr(projected_tt_vectors_1, 'projection_on') or \ - not hasattr(projected_tt_vectors_2, 'projection_on'): - raise ValueError('Both arguments should be projections on the tangent ' - 'space of some other TT-object. All projection* functions ' - 'leave .projection_on field in the resulting TT-object ' - 'which is not present in the arguments you\'ve provided') - - if projected_tt_vectors_1.projection_on != projected_tt_vectors_2.projection_on: - raise ValueError('Both arguments should be projections on the tangent ' - 'space of the same TT-object. The provided arguments are ' - 'projections on different TT-objects (%s and %s). Or at ' - 'least the pointers are different.' % - (projected_tt_vectors_1.projection_on, - projected_tt_vectors_2.projection_on)) - - # Always work with batches of objects for simplicity. - projected_tt_vectors_1 = shapes.expand_batch_dim(projected_tt_vectors_1) - projected_tt_vectors_2 = shapes.expand_batch_dim(projected_tt_vectors_2) - - ndims = projected_tt_vectors_1.ndims() - tt_ranks = shapes.lazy_tt_ranks(projected_tt_vectors_1) - - if projected_tt_vectors_1.is_tt_matrix(): - right_size = tt_ranks[1] // 2 - curr_core_1 = projected_tt_vectors_1.tt_cores[0] - curr_core_2 = projected_tt_vectors_2.tt_cores[0] - curr_du_1 = curr_core_1[:, :, :, :, :right_size] - curr_du_2 = curr_core_2[:, :, :, :, :right_size] - res = tf.einsum('paijb,qaijb->pq', curr_du_1, curr_du_2) - for core_idx in range(1, ndims): - left_size = tt_ranks[core_idx] // 2 - right_size = tt_ranks[core_idx + 1] // 2 - curr_core_1 = projected_tt_vectors_1.tt_cores[core_idx] - curr_core_2 = projected_tt_vectors_2.tt_cores[core_idx] - curr_du_1 = curr_core_1[:, left_size:, :, :, :right_size] - curr_du_2 = curr_core_2[:, left_size:, :, :, :right_size] - res += tf.einsum('paijb,qaijb->pq', curr_du_1, curr_du_2) - - left_size = tt_ranks[-2] // 2 - curr_core_1 = projected_tt_vectors_1.tt_cores[-1] - curr_core_2 = projected_tt_vectors_2.tt_cores[-1] - curr_du_1 = curr_core_1[:, left_size:, :, :, :] - curr_du_2 = curr_core_2[:, left_size:, :, :, :] - res += tf.einsum('paijb,qaijb->pq', curr_du_1, curr_du_2) - else: - # Working with TT-tensor, not TT-matrix. - right_size = tt_ranks[1] // 2 - curr_core_1 = projected_tt_vectors_1.tt_cores[0] - curr_core_2 = projected_tt_vectors_2.tt_cores[0] - curr_du_1 = curr_core_1[:, :, :, :right_size] - curr_du_2 = curr_core_2[:, :, :, :right_size] - res = tf.einsum('paib,qaib->pq', curr_du_1, curr_du_2) - for core_idx in range(1, ndims): - left_size = tt_ranks[core_idx] // 2 - right_size = tt_ranks[core_idx + 1] // 2 - curr_core_1 = projected_tt_vectors_1.tt_cores[core_idx] - curr_core_2 = projected_tt_vectors_2.tt_cores[core_idx] - curr_du_1 = curr_core_1[:, left_size:, :, :right_size] - curr_du_2 = curr_core_2[:, left_size:, :, :right_size] - res += tf.einsum('paib,qaib->pq', curr_du_1, curr_du_2) - - left_size = tt_ranks[-2] // 2 - curr_core_1 = projected_tt_vectors_1.tt_cores[-1] - curr_core_2 = projected_tt_vectors_2.tt_cores[-1] - curr_du_1 = curr_core_1[:, left_size:, :, :] - curr_du_2 = curr_core_2[:, left_size:, :, :] - res += tf.einsum('paib,qaib->pq', curr_du_1, curr_du_2) - return res - - -def add_n_projected(tt_objects, coef=None): - """Adds all input TT-objects that are projections on the same tangent space. - - add_projected((a, b)) is equivalent add(a, b) for a and b that are from the - same tangent space, but doesn't increase the TT-ranks. - - Args: - tt_objects: a list of TT-objects that are projections on the same tangent - space. - coef: a list of numbers or anything else convertable to tf.Tensor. - If provided, computes weighted sum. The size of this array should be - len(tt_objects) x tt_objects[0].batch_size - - Returns: - TT-objects representing the sum of the tt_objects (weighted sum if coef is - provided). The TT-rank of the result equals to the TT-ranks of the arguments. - """ - for tt in tt_objects: - if not hasattr(tt, 'projection_on'): - raise ValueError('Both arguments should be projections on the tangent ' - 'space of some other TT-object. All projection* functions ' - 'leave .projection_on field in the resulting TT-object ' - 'which is not present in the argument you\'ve provided.') - - projection_on = tt_objects[0].projection_on - for tt in tt_objects[1:]: - if tt.projection_on != projection_on: - raise ValueError('All tt_objects should be projections on the tangent ' - 'space of the same TT-object. The provided arguments are ' - 'projections on different TT-objects (%s and %s). Or at ' - 'least the pointers are different.' % (tt.projection_on, - projection_on)) - if coef is not None: - coef = tf.convert_to_tensor(coef) - if coef.get_shape().ndims > 1: - # In batch case we will need to multiply each core by this coefficients - # along the first axis. To do it need to reshape the coefs to match - # the TT-cores number of dimensions. - some_core = tt_objects[0].tt_cores[0] - dim_array = [1] * (some_core.get_shape().ndims + 1) - dim_array[0] = coef.get_shape()[0].value - dim_array[1] = coef.get_shape()[1].value - coef = tf.reshape(coef, dim_array) - - ndims = tt_objects[0].ndims() - tt_ranks = shapes.lazy_tt_ranks(tt_objects[0]) - left_rank_dim = tt_objects[0].left_tt_rank_dim - right_rank_dim = tt_objects[0].right_tt_rank_dim - res_cores = [] - - def slice_tt_core(tt_core, left_idx, right_idx): - num_tt_core_dims = len(tt_core.get_shape()) - idx = [slice(None)] * num_tt_core_dims - idx[left_rank_dim] = left_idx - idx[right_rank_dim] = right_idx - return tt_core[idx] - - right_half_rank = tt_ranks[1] // 2 - left_chunks = [] - for obj_idx, tt in enumerate(tt_objects): - curr_core = slice_tt_core(tt.tt_cores[0], slice(None), - slice(0, right_half_rank)) - if coef is not None: - curr_core *= coef[obj_idx] - left_chunks.append(curr_core) - left_part = tf.add_n(left_chunks) - first_obj_core = tt_objects[0].tt_cores[0] - right_part = slice_tt_core(first_obj_core, slice(None), - slice(right_half_rank, None)) - first_core = tf.concat((left_part, right_part), axis=right_rank_dim) - res_cores.append(first_core) - - for core_idx in range(1, ndims - 1): - first_obj_core = tt_objects[0].tt_cores[core_idx] - left_half_rank = tt_ranks[core_idx] // 2 - right_half_rank = tt_ranks[core_idx + 1] // 2 - - upper_part = slice_tt_core(tt.tt_cores[core_idx], slice(0, left_half_rank), - slice(None)) - lower_right_part = slice_tt_core(first_obj_core, - slice(left_half_rank, None), - slice(right_half_rank, None)) - - lower_left_chunks = [] - for obj_idx, tt in enumerate(tt_objects): - curr_core = slice_tt_core(tt.tt_cores[core_idx], - slice(left_half_rank, None), - slice(0, right_half_rank)) - if coef is not None: - curr_core *= coef[obj_idx] - lower_left_chunks.append(curr_core) - lower_left_part = tf.add_n(lower_left_chunks) - lower_part = tf.concat((lower_left_part, lower_right_part), - axis=right_rank_dim) - curr_core = tf.concat((upper_part, lower_part), axis=left_rank_dim) - res_cores.append(curr_core) - - left_half_rank = tt_ranks[ndims - 1] // 2 - upper_part = slice_tt_core(tt.tt_cores[-1], slice(0, left_half_rank), - slice(None)) - lower_chunks = [] - for obj_idx, tt in enumerate(tt_objects): - curr_core = slice_tt_core(tt.tt_cores[-1], slice(left_half_rank, None), - slice(None)) - if coef is not None: - curr_core *= coef[obj_idx] - lower_chunks.append(curr_core) - lower_part = tf.add_n(lower_chunks) - last_core = tf.concat((upper_part, lower_part), axis=left_rank_dim) - res_cores.append(last_core) - - raw_shape = tt_objects[0].get_raw_shape() - static_tt_ranks = tt_objects[0].get_tt_ranks() - if isinstance(tt_objects[0], TensorTrain): - res = TensorTrain(res_cores, raw_shape, static_tt_ranks) - elif isinstance(tt_objects[0], TensorTrainBatch): - res = TensorTrainBatch(res_cores, raw_shape, static_tt_ranks, - tt_objects[0].batch_size) - # Maintain the projection_on property. - res.projection_on = tt_objects[0].projection_on - return res diff --git a/build/lib/t3f/riemannian_test.py b/build/lib/t3f/riemannian_test.py deleted file mode 100644 index 25ddb7d1..00000000 --- a/build/lib/t3f/riemannian_test.py +++ /dev/null @@ -1,250 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f.tensor_train import TensorTrain -from t3f import ops -from t3f import initializers -from t3f import riemannian -from t3f import shapes -from t3f import batch_ops - - -class RiemannianTest(tf.test.TestCase): - - def testProjectOnItself(self): - # Projection of X into the tangent space of itself is X: P_x(x) = x. - tens = initializers.random_tensor((2, 3, 4)) - proj = riemannian.project_sum(tens, tens) - with self.test_session() as sess: - actual_val, desired_val = sess.run((ops.full(proj), ops.full(tens))) - self.assertAllClose(desired_val, actual_val) - - def testProject(self): - # Compare our projection with the results obtained (and precomputed) from - # tt.riemannian.project which is well tested. - tangent_tens_cores = ([[[-0.42095269, 0.02130842], - [-0.4181081 , 0.42945687], - [ 0.45972439, -0.4525616 ], - [-0.17159869, -0.14505528]]], [[[ 0.23344421], - [ 0.81480049], - [-0.92385135]], - - [[-0.19279465], - [ 0.524976 ], - [-0.40149197]]]) - tangent_tens = TensorTrain(tangent_tens_cores, (4, 3), (1, 2, 1)) - tens_cores = ([[[-1.01761142, 0.36075896, -0.2493624 ], - [-0.99896565, -1.12685474, 1.02832458], - [ 1.08739724, -0.6537435 , 1.99975537], - [ 0.35128005, 0.40395104, -0.16790072]]], [[[ 0.34105142], - [ 0.10371947], - [-1.76464904]], - - [[ 0.32639758], - [-1.27843174], - [-1.41590327]], - - [[ 0.76616274], - [ 0.6577514 ], - [ 2.13703185]]]) - tens = TensorTrain(tens_cores, (4, 3), (1, 3, 1)) - desired_projection = [[-0.67638254, -1.17163914, 0.29850939], - [-1.66479093, -0.99003251, 2.46629195], - [-0.04847773, -0.72908174, 0.20142675], - [ 0.34431125, -0.20935516, -1.15864246]] - proj = riemannian.project_sum(tens, tangent_tens) - with self.test_session() as sess: - self.assertAllClose(desired_projection, ops.full(proj).eval()) - - def testProjectSum(self): - # Test projecting a batch of TT-tensors. - tens = initializers.random_tensor_batch((2, 3, 4), batch_size=3) - tangent_tens = initializers.random_tensor((2, 3, 4), 3) - weighted_sum = tens[0] + tens[1] + tens[2] - direct_proj = riemannian.project_sum(weighted_sum, tangent_tens) - actual_proj = riemannian.project_sum(tens, tangent_tens) - with self.test_session() as sess: - res = sess.run((ops.full(direct_proj), ops.full(actual_proj))) - desired_val, actual_val = res - self.assertAllClose(desired_val, actual_val) - - def testProjectWeightedSum(self): - # Test projecting a batch of TT-tensors with providing coefs. - tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4) - coef = [0.1, -2, 0, 0.4] - tangent_tens = initializers.random_tensor((2, 3, 4), 4) - weighted_sum = coef[0] * tens[0] + coef[1] * tens[1] + coef[2] * tens[2] - weighted_sum += coef[3] * tens[3] - direct_proj = riemannian.project_sum(weighted_sum, tangent_tens) - actual_proj = riemannian.project_sum(tens, tangent_tens, coef) - with self.test_session() as sess: - res = sess.run((ops.full(direct_proj), ops.full(actual_proj))) - desired_val, actual_val = res - self.assertAllClose(desired_val, actual_val) - - def testProjectWeightedSumMultipleOutputs(self): - # Test projecting a batch of TT-tensors with providing weights and outputing - # several TT objects with different weights. - tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4) - np.random.seed(0) - weights = np.random.randn(4, 2).astype(np.float32) - tangent_tens = initializers.random_tensor((2, 3, 4), 4) - weighted_sum_1 = weights[0, 0] * tens[0] + weights[1, 0] * tens[1] +\ - weights[2, 0] * tens[2] + weights[3, 0] * tens[3] - weighted_sum_2 = weights[0, 1] * tens[0] + weights[1, 1] * tens[1] +\ - weights[2, 1] * tens[2] + weights[3, 1] * tens[3] - direct_proj_1 = riemannian.project_sum(weighted_sum_1, tangent_tens) - direct_proj_2 = riemannian.project_sum(weighted_sum_2, tangent_tens) - direct_proj_1 = shapes.expand_batch_dim(direct_proj_1) - direct_proj_2 = shapes.expand_batch_dim(direct_proj_2) - direct_projs = batch_ops.concat_along_batch_dim((direct_proj_1, direct_proj_2)) - actual_proj = riemannian.project_sum(tens, tangent_tens, weights) - with self.test_session() as sess: - res = sess.run((ops.full(direct_projs), ops.full(actual_proj))) - desired_val, actual_val = res - self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) - - def testProjectMatrixOnItself(self): - # Project a TT-matrix on itself. - # Projection of X into the tangent space of itself is X: P_x(x) = x. - tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4))) - proj = riemannian.project_sum(tt_mat, tt_mat) - with self.test_session() as sess: - actual_val, desired_val = sess.run((ops.full(proj), ops.full(tt_mat))) - self.assertAllClose(desired_val, actual_val) - - def testCompareProjectSumAndProject(self): - # Compare results of project_sum and project. - tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4) - tangent_tens = initializers.random_tensor((2, 3, 4), 4) - project_sum = riemannian.project_sum(tens, tangent_tens, tf.eye(4)) - project = riemannian.project(tens, tangent_tens) - with self.test_session() as sess: - res = sess.run((ops.full(project_sum), ops.full(project))) - project_sum_val, project_val = res - self.assertAllClose(project_sum_val, project_val) - - def testProjectMatmul(self): - # Project a TT-matrix times TT-vector on a TT-vector. - tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4))) - tt_vec_what = initializers.random_matrix_batch(((2, 3, 4), None), - batch_size=3) - tt_vec_where = initializers.random_matrix(((2, 3, 4), None)) - proj = riemannian.project_matmul(tt_vec_what, tt_vec_where, tt_mat) - matvec = ops.matmul(tt_mat, tt_vec_what) - proj_desired = riemannian.project(matvec, tt_vec_where) - with self.test_session() as sess: - actual_val, desired_val = sess.run((ops.full(proj), ops.full(proj_desired))) - self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) - - def testPairwiseFlatInnerTensor(self): - # Compare pairwise_flat_inner_projected against naive implementation. - what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3) - what2 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=4) - where = initializers.random_tensor((2, 3, 4), 3) - projected1 = riemannian.project(what1, where) - projected2 = riemannian.project(what2, where) - desired = batch_ops.pairwise_flat_inner(projected1, projected2) - actual = riemannian.pairwise_flat_inner_projected(projected1, projected2) - with self.test_session() as sess: - desired_val, actual_val = sess.run((desired, actual)) - self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) - - with self.assertRaises(ValueError): - # Second argument is not a projection on the tangent space. - riemannian.pairwise_flat_inner_projected(projected1, what2) - where2 = initializers.random_tensor((2, 3, 4), 3) - another_projected2 = riemannian.project(what2, where2) - with self.assertRaises(ValueError): - # The arguments are projections on different tangent spaces. - riemannian.pairwise_flat_inner_projected(projected1, another_projected2) - - def testPairwiseFlatInnerMatrix(self): - # Compare pairwise_flat_inner_projected against naive implementation. - what1 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=3) - what2 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=4) - where = initializers.random_matrix(((2, 3, 4), None), 3) - projected1 = riemannian.project(what1, where) - projected2 = riemannian.project(what2, where) - desired = batch_ops.pairwise_flat_inner(projected1, projected2) - actual = riemannian.pairwise_flat_inner_projected(projected1, projected2) - with self.test_session() as sess: - desired_val, actual_val = sess.run((desired, actual)) - self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) - - with self.assertRaises(ValueError): - # Second argument is not a projection on the tangent space. - riemannian.pairwise_flat_inner_projected(projected1, what2) - where2 = initializers.random_matrix(((2, 3, 4), None), 3) - another_projected2 = riemannian.project(what2, where2) - with self.assertRaises(ValueError): - # The arguments are projections on different tangent spaces. - riemannian.pairwise_flat_inner_projected(projected1, another_projected2) - - def testAddNProjected(self): - # Add several TT-objects from the same tangent space. - what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3) - what2 = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=3) - where = initializers.random_tensor((2, 3, 4), 3) - projected1 = riemannian.project(what1, where) - projected2 = riemannian.project(what2, where) - desired = ops.full(projected1 + projected2) - actual = ops.full(riemannian.add_n_projected((projected1, projected2))) - with self.test_session() as sess: - desired_val, actual_val = sess.run((desired, actual)) - self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) - - with self.assertRaises(ValueError): - # Second argument is not a projection on the tangent space. - riemannian.add_n_projected((projected1, what2)) - where2 = initializers.random_tensor((2, 3, 4), 3) - another_projected2 = riemannian.project(what2, where2) - with self.assertRaises(ValueError): - # The arguments are projections on different tangent spaces. - riemannian.add_n_projected((projected1, another_projected2)) - - def testWeightedAddNProjected(self): - # Add several TT-objects from the same tangent space with coefs. - what1 = initializers.random_tensor((2, 3, 4), 4) - what2 = initializers.random_tensor((2, 3, 4), 1) - where = initializers.random_tensor((2, 3, 4), 3) - projected1 = riemannian.project(what1, where) - projected2 = riemannian.project(what2, where) - desired = ops.full(1.2 * projected1 + -2.0 * projected2) - actual = ops.full(riemannian.add_n_projected((projected1, projected2), - coef=[1.2, -2.0])) - with self.test_session() as sess: - desired_val, actual_val = sess.run((desired, actual)) - self.assertAllClose(desired_val, actual_val) - - with self.assertRaises(ValueError): - # Second argument is not a projection on the tangent space. - riemannian.add_n_projected((projected1, what2), coef=[1.2, -2.0]) - where2 = initializers.random_tensor((2, 3, 4), 3) - another_projected2 = riemannian.project(what2, where2) - with self.assertRaises(ValueError): - # The arguments are projections on different tangent spaces. - riemannian.add_n_projected((projected1, another_projected2), - coef=[1.2, -2.0]) - - def testWeightedAddNProjectedBatch(self): - # Add several TT-batches from the same tangent space with coefs. - what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3) - what2 = initializers.random_tensor_batch((2, 3, 4), 1, batch_size=3) - where = initializers.random_tensor((2, 3, 4), 3) - projected1 = riemannian.project(what1, where) - projected2 = riemannian.project(what2, where) - - desired_0 = ops.full(1.2 * projected1[0] + -2.0 * projected2[0]) - desired_1 = ops.full(1.9 * projected1[1] + 2.0 * projected2[1]) - desired_2 = ops.full(0.0 * projected1[2] + 1.0 * projected2[2]) - desired = tf.stack((desired_0, desired_1, desired_2), axis=0) - actual = ops.full(riemannian.add_n_projected((projected1, projected2), - coef=[[1.2, 1.9, 0.0], - [-2.0, 2.0, 1.0]])) - with self.test_session() as sess: - desired_val, actual_val = sess.run((desired, actual)) - self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/shapes.py b/build/lib/t3f/shapes.py deleted file mode 100644 index bc411092..00000000 --- a/build/lib/t3f/shapes.py +++ /dev/null @@ -1,281 +0,0 @@ -import numpy as np -import tensorflow as tf - - -# TODO: test all these functions. -def tt_ranks(tt): - """Returns the TT-ranks of a TensorTrain. - - This operation returns a 1-D integer tensor representing the TT-ranks of - the input. - - Args: - tt: `TensorTrain` or `TensorTrainBatch` object. - - Returns: - A `Tensor` - """ - num_dims = tt.ndims() - ranks = [] - for i in range(num_dims): - ranks.append(tf.shape(tt.tt_cores[i])[tt.left_tt_rank_dim]) - ranks.append(tf.shape(tt.tt_cores[-1])[-1]) - return tf.stack(ranks, axis=0) - - -def shape(tt): - """Returns the shape of a TensorTrain. - - This operation returns a 1-D integer tensor representing the shape of - the input. For TT-matrices the shape would have two values, see raw_shape for - the tensor shape. - If the input is a TensorTrainBatch, the first dimension of the output is the - batch_size. - - Args: - tt: `TensorTrain` or `TensorTrainBatch` object. - - Returns: - A `Tensor` - """ - tt_raw_shape = raw_shape(tt) - if tt.is_tt_matrix(): - res = tf.reduce_prod(tt_raw_shape, axis=1) - else: - res = tt_raw_shape[0] - - # TODO: ugly. - from t3f.tensor_train_batch import TensorTrainBatch - if isinstance(tt, TensorTrainBatch): - res = tf.concat((tf.expand_dims(batch_size(tt), 0), res), axis=0) - - return res - - -def raw_shape(tt): - """Returns the shape of a TensorTrain. - - This operation returns a 2-D integer tensor representing the shape of - the input. - If the input is a TT-tensor, the shape will have 1 x ndims() elements. - If the input is a TT-matrix, the shape will have 2 x ndims() elements - representing the underlying tensor shape of the matrix. - - Args: - tt: `TensorTrain` or `TensorTrainBatch` object. - - Returns: - A 2-D `Tensor` of size 1 x ndims() or 2 x ndims() - """ - num_dims = tt.ndims() - num_tensor_axis = len(tt.get_raw_shape()) - final_raw_shape = [] - # TODO: ugly. - from t3f.tensor_train import TensorTrain - axes_shift = 1 if isinstance(tt, TensorTrain) else 2 - for ax in range(num_tensor_axis): - curr_raw_shape = [] - for core_idx in range(num_dims): - curr_raw_shape.append(tf.shape(tt.tt_cores[core_idx])[ax + axes_shift]) - final_raw_shape.append(tf.stack(curr_raw_shape, axis=0)) - return tf.stack(final_raw_shape, axis=0) - - -def batch_size(tt): - """Return the number of elements in a TensorTrainBatch. - - Return 0-D integer tensor. - - Raises: - ValueError if got `TensorTrain` which doesn't have batch_size as input.""" - if not hasattr(tt, 'batch_size'): - raise ValueError('batch size is not available for a TensorTrain object.') - first_core = tt.tt_cores[0] - # The first dimension of any TT-core in TensorTrainBatch is the batch size. - return tf.shape(first_core)[0] - - -def lazy_tt_ranks(tt): - """Returns static TT-ranks of a TensorTrain if defined, and dynamic otherwise. - - This operation returns a 1-D integer numpy array of TT-ranks if they are - available on the graph compilation stage and 1-D integer tensor of dynamic - TT-ranks otherwise. - - Args: - tt: `TensorTrain` object. - - Returns: - A 1-D numpy array or `tf.Tensor` - """ - static_tt_ranks = tt.get_tt_ranks() - if static_tt_ranks.is_fully_defined(): - return np.array(static_tt_ranks.as_list()) - else: - return tt_ranks(tt) - - -def lazy_shape(tt): - """Returns static shape of a TensorTrain if defined, and dynamic otherwise. - - This operation returns a 1-D integer numpy array representing the shape of the - input if it is available on the graph compilation stage and 1-D integer tensor - of dynamic shape otherwise. - - Args: - tt: `TensorTrain` object. - - Returns: - A 1-D numpy array or `tf.Tensor` - """ - static_shape = tt.get_shape() - if static_shape.is_fully_defined(): - return np.array(static_shape.as_list()) - else: - return shape(tt) - - -def lazy_raw_shape(tt): - """Returns static raw shape of a TensorTrain if defined, and dynamic otherwise. - - This operation returns a 2-D integer numpy array representing the raw shape of - the input if it is available on the graph compilation stage and 2-D integer - tensor of dynamic shape otherwise. - If the input is a TT-tensor, the raw shape will have 1 x ndims() elements. - If the input is a TT-matrix, the raw shape will have 2 x ndims() elements - representing the underlying tensor shape of the matrix. - - Args: - tt: `TensorTrain` object. - - Returns: - A 2-D numpy array or `tf.Tensor` of size 1 x ndims() or 2 x ndims() - """ - # If get_shape is fully defined, it guaranties that all elements of raw shape - # are defined. - if tt.get_shape().is_fully_defined(): - return np.array([s.as_list() for s in tt.get_raw_shape()]) - else: - return raw_shape(tt) - - -def lazy_batch_size(tt): - """Return static batch_size if available and dynamic otherwise. - - Args: - tt: `TensorTrainBatch` object. - - Returns: - A number or a 0-D `tf.Tensor` - - Raises: - ValueError if got `TensorTrain` which doesn't have batch_size as input.""" - if not hasattr(tt, 'batch_size'): - raise ValueError('batch size is not available for a TensorTrain object.') - if tt.batch_size is not None: - return tt.batch_size - else: - return batch_size(tt) - - -def clean_raw_shape(shape): - """Returns a tuple of TensorShapes for any valid shape representation. - - Args: - shape: An np.array, a tf.TensorShape (for tensors), a tuple of - tf.TensorShapes (for TT-matrices or tensors), or None - - Returns: - A tuple of tf.TensorShape, or None if the input is None - """ - if shape is None: - return None - if isinstance(shape, tf.TensorShape) or isinstance(shape[0], tf.TensorShape): - # Assume tf.TensorShape. - if isinstance(shape, tf.TensorShape): - shape = tuple((shape,)) - else: - np_shape = np.array(shape) - # Make sure that the shape is 2-d array both for tensors and TT-matrices. - np_shape = np.squeeze(np_shape) - if len(np_shape.shape) == 1: - # A tensor. - np_shape = [np_shape] - shape = [] - for i in range(len(np_shape)): - shape.append(tf.TensorShape(np_shape[i])) - shape = tuple(shape) - return shape - - -def is_batch_broadcasting_possible(tt_a, tt_b): - """Check that the batch broadcasting possible for the given batch sizes. - - Returns true if the batch sizes are the same or if one of them is 1. - - If the batch size that is supposed to be 1 is not known on compilation stage, - broadcasting is not allowed. - - Args: - tt_a: TensorTrain or TensorTrainBatch - tt_b: TensorTrain or TensorTrainBatch - - Returns: - Bool - """ - try: - if tt_a.batch_size is None and tt_b.batch_size is None: - # If both batch sizes are not available on the compilation stage, - # we cannot say if broadcasting is possible so we will not allow it. - return False - if tt_a.batch_size == tt_b.batch_size: - return True - if tt_a.batch_size == 1 or tt_b.batch_size == 1: - return True - return False - except AttributeError: - # One or both of the arguments are not batch tensor, but single TT tensors. - # In this case broadcasting is always possible. - return True - - -def squeeze_batch_dim(tt): - """Converts batch size 1 TensorTrainBatch into TensorTrain. - - Args: - tt: TensorTrain or TensorTrainBatch. - - Returns: - TensorTrain if the input is a TensorTrainBatch with batch_size == 1 (known - at compilation stage) or a TensorTrain. - TensorTrainBatch otherwise. - """ - try: - if tt.batch_size == 1: - return tt[0] - else: - return tt - except AttributeError: - # tt object does not have attribute batch_size, probably already - # a TensorTrain. - return tt - - -def expand_batch_dim(tt): - """Creates a 1-element TensorTrainBatch from a TensorTrain. - - Args: - tt: TensorTrain or TensorTrainBatch. - - Returns: - TensorTrainBatch - """ - if hasattr(tt, 'batch_size'): - return tt - else: - from t3f.tensor_train_batch import TensorTrainBatch - tt_cores = [] - for core_idx in range(tt.ndims()): - tt_cores.append(tf.expand_dims(tt.tt_cores[core_idx], 0)) - return TensorTrainBatch(tt_cores, tt.get_raw_shape(), tt.get_tt_ranks(), - batch_size=1) diff --git a/build/lib/t3f/tensor_train.py b/build/lib/t3f/tensor_train.py deleted file mode 100644 index b884e92f..00000000 --- a/build/lib/t3f/tensor_train.py +++ /dev/null @@ -1,229 +0,0 @@ -import tensorflow as tf - -from t3f.tensor_train_base import TensorTrainBase -from t3f import shapes - - -class TensorTrain(TensorTrainBase): - """Represents a Tensor Train object (a TT-tensor or TT-matrix). - - t3f represents a Tensor Train object as a tuple of TT-cores. - ``` - @@__init__ - @@get_raw_shape - @@get_shape - @@tt_cores - @@dtype - @@name - @@graph - @@ndims - @@get_tt_ranks - @@left_tt_rank_dim - @@right_tt_rank_dim - @@is_tt_matrix - @@is_variable - @@eval - """ - - def __init__(self, tt_cores, shape=None, tt_ranks=None, convert_to_tensors=True): - """Creates a `TensorTrain`. - - Args: - tt_cores: A tuple of 3d or 4d tensor-like objects of shape - `[r_k-1, n_k, r_k]`. - Tensor-like can be numpy array, tf.Tensor, of tf.Variable - shape: Shape of the underlying tensor. If None, tries to infer from the - cores (not always possible even if it should be, e.g. if ranks are - unknown, than the whole shape of a core can be unknown). - tt_ranks: a TensorShape of length d+1 (d is the dimensionality of - the underlying tensor). The first and the last ranks are assumed to - equal to 1. If None, tries to infer the ranks from the cores. - convert_to_tensors: bool, if True than convert each element of the - tt_cores tuple into a tf.Tensor (e.g. to initialize from np.array) - - Returns: - A `TensorTrain`. - - Raises: - ValueError if the provided TT-cores are not valid or inconsistent with - the provided shape. - """ - tt_cores = list(tt_cores) - if convert_to_tensors: - # TODO: what does this namescope do? - with tf.name_scope("TensorTrain", tt_cores): - for i in range(len(tt_cores)): - name = "core%d" % i - tt_cores[i] = tf.convert_to_tensor(tt_cores[i], name=name) - - if not _are_tt_cores_valid(tt_cores, shape, tt_ranks): - raise ValueError('The tt_cores provided to TensorTrain constructor are ' - 'not valid, have different dtypes, or are inconsistent ' - 'with the provided shape or TT-ranks.') - - self._tt_cores = tuple(tt_cores) - self._raw_shape = shapes.clean_raw_shape(shape) - if self._raw_shape is None: - self._raw_shape = _infer_raw_shape(self._tt_cores) - self._tt_ranks = None if tt_ranks is None else tf.TensorShape(tt_ranks) - if self._tt_ranks is None: - self._tt_ranks = _infer_tt_ranks(self._tt_cores) - - @property - def tt_cores(self): - """A tuple of TT-cores. - - Returns: - A tuple of 3d or 4d tensors shape - `[r_k-1, n_k, r_k]` - or - `[r_k-1, n_k, m_k, r_k]` - """ - return self._tt_cores - - @property - def left_tt_rank_dim(self): - """The dimension of the left TT-rank in each TT-core.""" - return 0 - - @property - def right_tt_rank_dim(self): - """The dimension of the right TT-rank in each TT-core.""" - if self.is_tt_matrix(): - # The dimensions of each TT-core are - # [left_rank, n, m, right_rank] - return 3 - else: - # The dimensions of each TT-core are - # [left_rank, n, right_rank] - return 2 - - def __str__(self): - """A string describing the TensorTrain object, its TT-rank, and shape.""" - shape = self.get_shape() - tt_ranks = self.get_tt_ranks() - variable_str = ' variable' if self.is_variable() else '' - if self.is_tt_matrix(): - raw_shape = self.get_raw_shape() - return "A TT-Matrix%s of size %d x %d, underlying tensor " \ - "shape: %s x %s, TT-ranks: %s" % (variable_str, shape[0], shape[1], - raw_shape[0], raw_shape[1], - tt_ranks) - else: - return "A Tensor Train%s of shape %s, TT-ranks: %s" % (variable_str, - shape, tt_ranks) - - def __getitem__(self, slice_spec): - """Basic indexing, returns a `TensorTrain` containing the specified region. - - Examples: - >>> a = t3f.random_tensor((2, 3, 4)) - >>> a[1, :, :] - is a 2D TensorTrain 3 x 4. - >>> a[1:2, :, :] - is a 3D TensorTrain 1 x 3 x 4 - """ - if len(slice_spec) != self.ndims(): - raise ValueError('Expected %d indices, got %d' % (self.ndims(), - len(slice_spec))) - new_tt_cores = [] - remainder = None - for i in range(self.ndims()): - curr_core = self.tt_cores[i] - if self.is_tt_matrix(): - raise NotImplementedError - else: - sliced_core = curr_core[:, slice_spec[i], :] - if len(curr_core.get_shape()) != len(sliced_core.get_shape()): - # This index is specified exactly and we want to collapse this axis. - if remainder is None: - remainder = sliced_core - else: - remainder = tf.matmul(remainder, sliced_core) - else: - if remainder is not None: - # Add reminder from the previous collapsed cores to the current - # core. - sliced_core = tf.einsum('ab,bid->aid', remainder, sliced_core) - remainder = None - new_tt_cores.append(sliced_core) - - if remainder is not None: - # The reminder obtained from collapsing the last cores. - new_tt_cores[-1] = tf.einsum('aib,bd->aid', new_tt_cores[-1], remainder) - remainder = None - # TODO: infer the output ranks and shape. - return TensorTrain(new_tt_cores) - - -def _are_tt_cores_valid(tt_cores, shape, tt_ranks): - """Check if dimensions of the TT-cores are consistent and the dtypes coincide. - - Args: - tt_cores: a tuple of `Tensor` objects - shape: An np.array, a tf.TensorShape (for tensors), a tuple of - tf.TensorShapes (for TT-matrices or tensors), or None - tt_ranks: An np.array or a tf.TensorShape of length len(tt_cores)+1. - - Returns: - boolean, True if the dimensions and dtypes are consistent. - """ - shape = shapes.clean_raw_shape(shape) - num_dims = len(tt_cores) - - for core_idx in range(1, num_dims): - if tt_cores[core_idx].dtype != tt_cores[0].dtype: - return False - try: - for core_idx in range(num_dims): - curr_core_shape = tt_cores[core_idx].get_shape() - if len(curr_core_shape) != len(tt_cores[0].get_shape()): - # Shapes are inconsistent. - return False - if shape is not None: - for i in range(len(shape)): - if curr_core_shape[i + 1] != shape[i][core_idx]: - # The TT-cores are not aligned with the given shape. - return False - if core_idx >= 1: - prev_core_shape = tt_cores[core_idx - 1].get_shape() - if curr_core_shape[0] != prev_core_shape[-1]: - # TT-ranks are inconsistent. - return False - if tt_ranks is not None: - if curr_core_shape[0] != tt_ranks[core_idx]: - # The TT-ranks are not aligned with the TT-cores shape. - return False - if curr_core_shape[-1] != tt_ranks[core_idx + 1]: - # The TT-ranks are not aligned with the TT-cores shape. - return False - if tt_cores[0].get_shape()[0] != 1 or tt_cores[-1].get_shape()[-1] != 1: - # The first or the last rank is not 1. - return False - except ValueError: - # The shape of the TT-cores is undetermined, can not validate it. - pass - return True - - -def _infer_raw_shape(tt_cores): - """Tries to infer the (static) raw shape from the TT-cores.""" - num_dims = len(tt_cores) - num_tensor_shapes = len(tt_cores[0].get_shape().as_list()) - 2 - raw_shape = [[] for _ in range(num_tensor_shapes)] - for dim in range(num_dims): - curr_core_shape = tt_cores[dim].get_shape() - for i in range(num_tensor_shapes): - raw_shape[i].append(curr_core_shape[i + 1]) - for i in range(num_tensor_shapes): - raw_shape[i] = tf.TensorShape(raw_shape[i]) - return tuple(raw_shape) - - -def _infer_tt_ranks(tt_cores): - """Tries to infer the (static) raw shape from the TT-cores.""" - tt_ranks = [] - for i in range(len(tt_cores)): - tt_ranks.append(tt_cores[i].get_shape()[0]) - tt_ranks.append(tt_cores[-1].get_shape()[-1]) - return tf.TensorShape(tt_ranks) diff --git a/build/lib/t3f/tensor_train_base.py b/build/lib/t3f/tensor_train_base.py deleted file mode 100644 index 7bad0a07..00000000 --- a/build/lib/t3f/tensor_train_base.py +++ /dev/null @@ -1,189 +0,0 @@ -import numpy as np -import tensorflow as tf - - -# TODO: check the methods of _TensorLike -class TensorTrainBase(object): - """An abstract class that represents a collection of Tensor Train cores. - ``` - @@__init__ - @@get_raw_shape - @@get_shape - @@tt_cores - @@dtype - @@name - @@graph - @@ndims - @@get_tt_ranks - @@is_tt_matrix - @@is_variable - @@eval - """ - - def __init__(self, tt_cores): - """Creates a `TensorTrainBase`.""" - pass - - def get_raw_shape(self): - """Get tuple of `TensorShapes` representing the shapes of the underlying TT-tensor. - - Tuple contains one `TensorShape` for TT-tensor and 2 `TensorShapes` for - TT-matrix - - Returns: - A tuple of `TensorShape` objects. - """ - return self._raw_shape - - def get_shape(self): - """Get the `TensorShape` representing the shape of the dense tensor. - - Returns: - A `TensorShape` object. - """ - raw_shape = self.get_raw_shape() - if self.is_tt_matrix(): - # TODO: as list is not available if shape is partly known. - m = np.prod(raw_shape[0].as_list()) - n = np.prod(raw_shape[1].as_list()) - return tf.TensorShape((m, n)) - else: - return raw_shape[0] - - @property - def tt_cores(self): - """A tuple of TT-cores.""" - return self._tt_cores - - @property - def dtype(self): - """The `DType` of elements in this tensor.""" - # TODO: where is this created? - return self.tt_cores[0].dtype - - @property - def name(self): - """The name of the TensorTrain. - - Returns: - String, the scope in which the TT-cores are defined. - """ - core_name = self.tt_cores[0].name - idx = core_name.rfind('/') - return core_name[:idx] - - @property - def graph(self): - """The `Graph` that contains the tt_cores tensors.""" - # TODO: check in init that the other cores are from the same graph. - return self.tt_cores[0].graph - - def __str__(self): - """A string describing the TensorTrain object, its TT-rank and shape.""" - return NotImplementedError - - def ndims(self): - """Get the number of dimensions of the underlying TT-tensor. - - Returns: - A number. - """ - return len(self.tt_cores) - - def get_tt_ranks(self): - """Get the TT-ranks in an array of size `num_dims`+1. - - The first and the last TT-rank are guarantied to be 1. - - Returns: - TensorShape of size `num_dims`+1. - """ - return self._tt_ranks - - def is_tt_matrix(self): - """Returns True if the TensorTrain object represents a TT-matrix.""" - return len(self.get_raw_shape()) == 2 - - def is_variable(self): - """True if the TensorTrain object is a variable (e.g. is trainable).""" - return isinstance(self.tt_cores[0], tf.Variable) - - @property - def op(self): - """The `Operation` that evaluates all the cores.""" - return tf.group(*[c.op for c in self.tt_cores]) - - def eval(self, feed_dict=None, session=None): - """Evaluates this sparse tensor in a `Session`. - - Calling this method will execute all preceding operations that - produce the inputs needed for the operation that produces this - tensor. - *N.B.* Before invoking `SparseTensor.eval()`, its graph must have been - launched in a session, and either a default session must be - available, or `session` must be specified explicitly. - - Args: - feed_dict: A dictionary that maps `Tensor` objects to feed values. - See [`Session.run()`](../../api_docs/python/client.md#Session.run) for a - description of the valid feed values. - session: (Optional.) The `Session` to be used to evaluate this sparse - tensor. If none, the default session will be used. - """ - # TODO: implement feed_dict - if session is None: - session = tf.get_default_session() - session.run(self.tt_cores) - - # TODO: do we need this? - # @staticmethod - # def _override_operator(operator, func): - # _override_helper(SparseTensor, operator, func) - - def __add__(self, other): - """Returns a TensorTrain corresponding to element-wise sum tt_a + tt_b. - - Supports broadcasting (e.g. you can add TensorTrainBatch and TensorTrain). - Just calls t3f.add, see its documentation for details. - """ - # TODO: ugly. - # We can't import ops in the beginning since it creates cyclic dependencies. - from t3f import ops - return ops.add(self, other) - - def __sub__(self, other): - """Returns a TensorTrain corresponding to element-wise difference tt_a - tt_b. - - Supports broadcasting (e.g. you can subtract TensorTrainBatch and - TensorTrain). - Just calls t3f.add(self, (-1) * other), see its documentation for details. - """ - # TODO: ugly. - # We can't import ops in the beginning since it creates cyclic dependencies. - from t3f import ops - return ops.add(self, ops.multiply(other, -1.)) - - def __neg__(self): - """Returns a TensorTrain corresponding to element-wise negative -tt_a. - - Just calls t3f.multiply(self, -1.), see its documentation for details. - """ - # TODO: ugly. - # We can't import ops in the beginning since it creates cyclic dependencies. - from t3f import ops - return ops.multiply(self, -1.) - - def __mul__(self, other): - """Returns a TensorTrain corresponding to element-wise product tt_a * tt_b. - - Supports broadcasting (e.g. you can multiply TensorTrainBatch and - TensorTrain). - Just calls t3f.multiply, see its documentation for details. - """ - # TODO: ugly. - # We can't import ops in the beginning since it creates cyclic dependencies. - from t3f import ops - return ops.multiply(self, other) - - # To support 'TT * 4' as well as '4 * TT'. - __rmul__ = __mul__ diff --git a/build/lib/t3f/tensor_train_batch.py b/build/lib/t3f/tensor_train_batch.py deleted file mode 100644 index 5ff0410b..00000000 --- a/build/lib/t3f/tensor_train_batch.py +++ /dev/null @@ -1,367 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f.tensor_train_base import TensorTrainBase -from t3f.tensor_train import TensorTrain -from t3f import shapes - - -class TensorTrainBatch(TensorTrainBase): - """Represents a batch of Tensor Train objects (TT-tensors or TT-matrices). - - t3f represents a Tensor Train object as a tuple of TT-cores. - ``` - @@__init__ - @@get_raw_shape - @@get_shape - @@tt_cores - @@dtype - @@name - @@graph - @@ndims - @@get_tt_ranks - @@left_tt_rank_dim - @@right_tt_rank_dim - @@is_tt_matrix - @@is_variable - @@eval - """ - - def __init__(self, tt_cores, shape=None, tt_ranks=None, batch_size=None, - convert_to_tensors=True): - """Creates a `TensorTrainBatch`. - - Args: - tt_cores: A tuple of 4d or 5d tensor-like objects of shape - `[batch_size, r_k-1, n_k, r_k]` or - `[batch_size, r_k-1, n_k, m_k, r_k]` - Tensor-like can be numpy array, tf.Tensor, of tf.Variable - batch_size: number of elements in the batch. If None, tries to infer from - the TT-cores (not always possible even if it should be, e.g. if ranks - are unknown, than the whole shape of a core can be unknown). - shape: Shape of the underlying tensor. If None, tries to infer from the - TT-cores. - tt_ranks: a TensorShape of length d+1 (d is the dimensionality of - the underlying tensor). The first and the last ranks are assumed to - equal to 1. If None, tries to infer the ranks from the cores. - convert_to_tensors: bool, if True than convert each element of the - tt_cores tuple into a tf.Tensor (e.g. to initialize from np.array) - - Returns: - A `TensorTrainBatch`. - - Raises: - ValueError if the provided TT-cores are not valid or inconsistent with - the provided shape. - """ - tt_cores = list(tt_cores) - if convert_to_tensors: - # TODO: what does this namescope do? - with tf.name_scope("TensorTrainBatch", tt_cores): - for i in range(len(tt_cores)): - name = "core%d" % i - tt_cores[i] = tf.convert_to_tensor(tt_cores[i], name=name) - - if not _are_batch_tt_cores_valid(tt_cores, shape, tt_ranks, batch_size): - raise ValueError('The tt_cores provided to TensorTrainBatch constructor ' - 'are not valid, have different dtypes, or are ' - 'inconsistent with the provided batch_size, shape, or ' - 'TT-ranks.') - - self._tt_cores = tuple(tt_cores) - if batch_size is None: - self._batch_size = tt_cores[0].get_shape()[0].value - else: - self._batch_size = batch_size - self._raw_shape = shapes.clean_raw_shape(shape) - if self._raw_shape is None: - self._raw_shape = _infer_batch_raw_shape(self._tt_cores) - self._tt_ranks = None if tt_ranks is None else tf.TensorShape(tt_ranks) - if self._tt_ranks is None: - self._tt_ranks = _infer_batch_tt_ranks(self._tt_cores) - - def get_shape(self): - """Get the `TensorShape` representing the shape of the dense tensor. - - The first dimension is the batch_size. - - Returns: - A `TensorShape` object. - """ - shape = TensorTrainBase.get_shape(self) - return tf.TensorShape(np.hstack((self.batch_size, shape))) - - @property - def tt_cores(self): - """A tuple of TT-cores. - - Returns: - A tuple of 4d or 5d tensors shape - `[batch_size, r_k-1, n_k, r_k]` - or - `[batch_size, r_k-1, n_k, m_k, r_k]` - """ - return self._tt_cores - - @property - def batch_size(self): - """The number of elements or None if not known.""" - return self._batch_size - - @property - def left_tt_rank_dim(self): - """The dimension of the left TT-rank in each TT-core.""" - return 1 - - @property - def right_tt_rank_dim(self): - """The dimension of the right TT-rank in each TT-core.""" - if self.is_tt_matrix(): - # The dimensions of each TT-core are - # [batch_idx, left_rank, n, m, right_rank] - return 4 - else: - # The dimensions of each TT-core are - # [batch_idx, left_rank, n, right_rank] - return 3 - - def __str__(self): - """A string describing the TensorTrainBatch, its TT-rank and shape.""" - shape = self.get_shape() - tt_ranks = self.get_tt_ranks() - - if self.batch_size is None: - batch_size_str = '(?)' - else: - batch_size_str = str(self.batch_size) - - if self.is_tt_matrix(): - raw_shape = self.get_raw_shape() - type_str = 'TT-matrix variables' if self.is_variable() else 'TT-matrices' - - return "A %s element batch of %s of size %d x %d, underlying tensor " \ - "shape: %s x %s, TT-ranks: %s" % (batch_size_str, type_str, - shape[1], shape[2], - raw_shape[0], raw_shape[1], - tt_ranks) - else: - if self.is_variable(): - type_str = 'Tensor Train variables' - else: - type_str = 'Tensor Trains' - return "A %s element batch of %s of shape %s, TT-ranks: %s" % \ - (batch_size_str, type_str, shape[1:], tt_ranks) - - @staticmethod - def _do_collapse_dim(slice_spec): - # Returns true if slice_spec is specified exactly and we want to collapse - # the corresponding axis, i.e. return an object with less dims. To be used - # in indexing functions. - # If its a actual slice, nothing to collapse. Otherwise (a number or - # a tf.Tensor) want to collapse. - return not isinstance(slice_spec, slice) - - def _batch_dim_getitem(self, element_spec): - """__getitem__ when provided only one (batch) index. - - Examples: - a[1] - a[1:3] - """ - - # This object index is specified exactly and we want to collapse the - # batch_size axis, i.e. return a TensorTrain instead of a TensorTrainBatch. - do_collapse_batch_dim = self._do_collapse_dim(element_spec) - - new_tt_cores = [] - for core_idx in range(self.ndims()): - curr_core = self.tt_cores[core_idx] - if self.is_tt_matrix(): - new_tt_cores.append(curr_core[element_spec, :, :, :, :]) - else: - new_tt_cores.append(curr_core[element_spec, :, :, :]) - if do_collapse_batch_dim: - # This index is specified exactly and we want to collapse the batch_size - # axis, i.e. return a TensorTrain instead of a TensorTrainBatch. - return TensorTrain(new_tt_cores, self.get_raw_shape(), - self.get_tt_ranks()) - else: - batch_size = new_tt_cores[0].get_shape()[0].value - return TensorTrainBatch(new_tt_cores, self.get_raw_shape(), - self.get_tt_ranks(), batch_size) - - def _full_getitem(self, slice_spec): - """__getitem__ when provided full index of length ndims + 1. - - Examples: - a = t3f.random_tensor_batch((2, 3, 4), batch_size=5) - a[:3, 1:2, 4, :] - """ - if len(slice_spec) != self.ndims() + 1: - raise ValueError('Expected %d indices, got %d' % (self.ndims() + 1, - len(slice_spec))) - # This object index is specified exactly and we want to collapse the - # batch_size axis, i.e. return a TensorTrain instead of a TensorTrainBatch. - do_collapse_batch_dim = self._do_collapse_dim(slice_spec[0]) - remainder = None - new_tt_cores = [] - for core_idx in range(self.ndims()): - curr_core = self.tt_cores[core_idx] - if self.is_tt_matrix(): - raise NotImplementedError - else: - sliced_core = curr_core[slice_spec[0], :, slice_spec[core_idx + 1], :] - do_collapse_curr_dim = self._do_collapse_dim(slice_spec[core_idx + 1]) - if do_collapse_curr_dim: - # This index is specified exactly and we want to collapse this axis. - if remainder is None: - remainder = sliced_core - else: - if do_collapse_batch_dim: - remainder = tf.einsum('ab,bd->ad', remainder, sliced_core) - else: - remainder = tf.einsum('oab,obd->oad', remainder, sliced_core) - else: - if remainder is not None: - # Add reminder from the previous collapsed cores to the current - # core. - if do_collapse_batch_dim: - sliced_core = tf.einsum('ab,bid->aid', remainder, sliced_core) - else: - sliced_core = tf.einsum('oab,obid->oaid', remainder, - sliced_core) - remainder = None - new_tt_cores.append(sliced_core) - - if remainder is not None: - # The reminder obtained from collapsing the last cores. - if do_collapse_batch_dim: - new_tt_cores[-1] = tf.einsum('aib,bd->aid', new_tt_cores[-1], - remainder) - - else: - new_tt_cores[-1] = tf.einsum('oaib,obd->oaid', new_tt_cores[-1], - remainder) - remainder = None - # TODO: infer the output ranks and shape. - if do_collapse_batch_dim: - return TensorTrain(new_tt_cores) - else: - return TensorTrainBatch(new_tt_cores) - - def __getitem__(self, slice_spec): - """Basic indexing, returns a `TensorTrainBatch` with the specified region. - - Examples: - >>> a = t3f.random_tensor_batch((2, 3, 4), batch_size=5) - >>> a[1:3, :, :, :] - is a 3D TensorTrainBatch 2 x 3 x 4 with batch_size = 2. - >>> a[1:3] - the same as above, a 3D TensorTrainBatch 2 x 3 x 4 with batch_size = 2. - >>> a[1, :, :, :] - is a 3D TensorTrain 2 x 3 x 4. - >>> a[1] - the same as above, a 3D TensorTrain 2 x 3 x 4. - >>> a[1:3, :, 1, :] - is a 2D TensorTrainBatch 2 x 4 with batch_size = 2. - >>> a[1, :, 1, :] - is a 2D TensorTrain 2 x 4. - - Returns: - `TensorTrainBatch` or `TensorTrain` depending on whether the first - (batch) dim was specified as a range or as a number. - """ - try: - slice_only_batch_dim = len(slice_spec) == 1 - except TypeError: - # The argument is not iterable, so it's a single slice, or a number, or a - # tf.Tensor with a number. - slice_only_batch_dim = True - - if slice_only_batch_dim: - # Indexing only for the batch_size axis, e.g. a[1:3]. - return self._batch_dim_getitem(slice_spec) - elif len(slice_spec) == self.ndims() + 1: - return self._full_getitem(slice_spec) - else: - raise ValueError('TensorTrainBatch.__getitem__: wrong number of ' - 'dimensions, expected 1 or %d, got %d' % - (self.ndims() + 1, len(slice_spec))) - - -def _are_batch_tt_cores_valid(tt_cores, shape, tt_ranks, batch_size): - """Check if dimensions of the TT-cores are consistent and the dtypes coincide. - - Args: - tt_cores: a tuple of `Tensor` objects - shape: An np.array, a tf.TensorShape (for tensors), a tuple of - tf.TensorShapes (for TT-matrices or tensors), or None - tt_ranks: An np.array or a tf.TensorShape of length len(tt_cores)+1. - batch_size: a number or None - - Returns: - boolean, True if the dimensions and dtypes are consistent. - """ - shape = shapes.clean_raw_shape(shape) - num_dims = len(tt_cores) - - for core_idx in range(1, num_dims): - if tt_cores[core_idx].dtype != tt_cores[0].dtype: - return False - try: - for core_idx in range(num_dims): - curr_core_shape = tt_cores[core_idx].get_shape() - if len(curr_core_shape) != len(tt_cores[0].get_shape()): - # Shapes are inconsistent. - return False - if batch_size is not None and curr_core_shape[0].value is not None: - if curr_core_shape[0].value != batch_size: - # The TT-cores are not aligned with the given batch_size. - return False - if shape is not None: - for i in range(len(shape)): - if curr_core_shape[i + 2] != shape[i][core_idx]: - # The TT-cores are not aligned with the given shape. - return False - if core_idx >= 1: - prev_core_shape = tt_cores[core_idx - 1].get_shape() - if curr_core_shape[1] != prev_core_shape[-1]: - # TT-ranks are inconsistent. - return False - if tt_ranks is not None: - if curr_core_shape[1] != tt_ranks[core_idx]: - # The TT-ranks are not aligned with the TT-cores shape. - return False - if curr_core_shape[-1] != tt_ranks[core_idx + 1]: - # The TT-ranks are not aligned with the TT-cores shape. - return False - if tt_cores[0].get_shape()[1] != 1 or tt_cores[-1].get_shape()[-1] != 1: - # The first or the last rank is not 1. - return False - except ValueError: - # The shape of the TT-cores is undetermined, can not validate it. - pass - return True - - -def _infer_batch_raw_shape(tt_cores): - """Tries to infer the (static) raw shape from the TT-cores.""" - num_dims = len(tt_cores) - num_tensor_shapes = len(tt_cores[0].get_shape().as_list()) - 3 - raw_shape = [[] for _ in range(num_tensor_shapes)] - for dim in range(num_dims): - curr_core_shape = tt_cores[dim].get_shape() - for i in range(num_tensor_shapes): - raw_shape[i].append(curr_core_shape[i + 2]) - for i in range(num_tensor_shapes): - raw_shape[i] = tf.TensorShape(raw_shape[i]) - return tuple(raw_shape) - - -def _infer_batch_tt_ranks(tt_cores): - """Tries to infer the (static) raw shape from the TT-cores.""" - tt_ranks = [] - for i in range(len(tt_cores)): - tt_ranks.append(tt_cores[i].get_shape()[1]) - tt_ranks.append(tt_cores[-1].get_shape()[-1]) - return tf.TensorShape(tt_ranks) diff --git a/build/lib/t3f/tensor_train_batch_test.py b/build/lib/t3f/tensor_train_batch_test.py deleted file mode 100644 index b609f1a7..00000000 --- a/build/lib/t3f/tensor_train_batch_test.py +++ /dev/null @@ -1,77 +0,0 @@ -import tensorflow as tf - -from t3f import initializers -from t3f import ops - - -class TensorTrainBatchTest(tf.test.TestCase): - - def testTensorIndexing(self): - tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3) - with self.test_session() as sess: - desired = ops.full(tens)[:, :, :, :] - actual = ops.full(tens[:, :, :, :]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[1:3, :, :, :] - actual = ops.full(tens[1:3]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[1, :, :, :] - actual = ops.full(tens[1]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[2, 1, :, :] - actual = ops.full(tens[2, 1, :, :]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[2, 1:2, 1, :] - actual = ops.full(tens[2, 1:2, 1, :]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[1:2, 0:3, :, 3] - actual = ops.full(tens[1:2, 0:3, :, 3]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[:, 1, :, 3] - actual = ops.full(tens[:, 1, :, 3]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - - # Wrong number of dims. - with self.assertRaises(ValueError): - tens[1, :, 3] - with self.assertRaises(ValueError): - tens[1, :, 3, 1:2, 1:3] - with self.assertRaises(ValueError): - tens[1, 1] - - def testPlaceholderTensorIndexing(self): - tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3) - with self.test_session() as sess: - start = tf.placeholder(tf.int32) - end = tf.placeholder(tf.int32) - - desired = ops.full(tens)[0:-1] - actual = ops.full(tens[start:end]) - desired, actual = sess.run([desired, actual], {start: 0, end: -1}) - self.assertAllClose(desired, actual) - - desired = ops.full(tens)[0:1] - actual = ops.full(tens[start:end]) - desired, actual = sess.run([desired, actual], {start: 0, end: 1}) - self.assertAllClose(desired, actual) - - desired = ops.full(tens)[1] - actual = ops.full(tens[start]) - desired, actual = sess.run([desired, actual], {start: 1}) - self.assertAllClose(desired, actual) - - desired = ops.full(tens)[1, 1:3, 1, :3] - actual = ops.full(tens[start, start:end, start, :end]) - desired, actual = sess.run([desired, actual], {start: 1, end: 3}) - self.assertAllClose(desired, actual) - - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/tensor_train_test.py b/build/lib/t3f/tensor_train_test.py deleted file mode 100644 index 22983424..00000000 --- a/build/lib/t3f/tensor_train_test.py +++ /dev/null @@ -1,138 +0,0 @@ -import tensorflow as tf - -from t3f import tensor_train -from t3f import initializers -from t3f import ops - - -class TensorTrainTest(tf.test.TestCase): - - def testValidateTTCores2d(self): - schedule = (((1, 1, 1, 1), (1, 1, 1), True), - ((1, 1, 1, 1), None, True), - ((1, 1, 1, 1), (1, 2, 1), False), - ((1, 1, 1, 1), (2, 1, 1), False), - ((1, 2, 2, 1), (1, 2, 1), True), - ((1, 2, 2, 1), (1, 1, 1), False), - ((1, 2, 2, 1), (1, 1, 2), False), - ((1, 2, 2, 1), None, True), - ((2, 1, 1, 1), None, False), - ((2, 1, 1, 1), (1, 1, 1), False), - ((1, 1, 1, 2), None, False), - ((1, 1, 1, 2), (1, 1, 1), False), - ((1, 1, 2, 1), None, False), - ((1, 1, 2, 1), (1, 2, 1), False), - ((1, 2, 1, 1), None, False), - ((1, 2, 1, 1), (1, 2, 1), False)) - - for tt_ranks, claimed_tt_ranks, desired in schedule: - a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1])) - b = tf.random_normal((tt_ranks[2], 9, tt_ranks[3])) - with self.test_session(): - actual = tensor_train._are_tt_cores_valid((a, b), (10, 9), - claimed_tt_ranks) - self.assertEqual(desired, actual) - # Wrong shape. - actual = tensor_train._are_tt_cores_valid((a, b), (9, 9), - claimed_tt_ranks) - self.assertEqual(False, actual) - if not desired: - with self.assertRaises(ValueError): - tensor_train.TensorTrain((a, b), (10, 9), claimed_tt_ranks) - - # Make dtypes inconsistent. - b_new = tf.cast(b, tf.float64) - actual = tensor_train._are_tt_cores_valid((a, b_new), (10, 9), - claimed_tt_ranks) - self.assertEqual(False, actual) - with self.assertRaises(ValueError): - tensor_train.TensorTrain((a, b_new), (10, 9), claimed_tt_ranks) - - def testValidateTTCores3d(self): - schedule = (((1, 1, 1, 1, 1, 1), (1, 1, 1, 1), True), - ((1, 1, 1, 1, 1, 1), None, True), - ((1, 1, 1, 1, 1, 1), (1, 1, 1, 2), False), - ((1, 1, 1, 1, 1, 1), (1, 1, 2, 1), False), - ((1, 2, 2, 2, 2, 1), (1, 2, 2, 1), True), - ((1, 2, 2, 2, 2, 1), None, True), - ((1, 2, 2, 2, 2, 1), (1, 2, 1, 1), False), - ((2, 1, 1, 1, 1, 1), None, False), - ((2, 1, 1, 1, 1, 1), (2, 1, 1, 1), False), - ((1, 1, 1, 1, 1, 2), None, False), - ((1, 1, 1, 1, 1, 2), (1, 1, 1, 2), False), - ((1, 1, 2, 1, 1, 1), None, False), - ((1, 1, 2, 1, 1, 1), (1, 2, 1, 1), False), - ((1, 2, 1, 1, 1, 1), None, False), - ((1, 2, 1, 1, 1, 1), (1, 2, 1, 1), False), - ((1, 2, 2, 1, 1, 1), (1, 2, 1, 1), True), - ((1, 2, 2, 1, 1, 1), (1, 2, 2, 1), False), - ((1, 2, 2, 1, 1, 1), None, True), - ((1, 2, 2, 3, 3, 1), (1, 2, 3, 1), True), - ((1, 2, 2, 3, 3, 1), None, True)) - - for tt_ranks, claimed_tt_ranks, desired in schedule: - a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1])) - b = tf.random_normal((tt_ranks[2], 1, tt_ranks[3])) - c = tf.random_normal((tt_ranks[4], 2, tt_ranks[5])) - with self.test_session(): - actual = tensor_train._are_tt_cores_valid((a, b, c), (10, 1, 2), - claimed_tt_ranks) - self.assertEqual(desired, actual) - # Wrong shape. - actual = tensor_train._are_tt_cores_valid((a, b, c), (10, 1, 1), - claimed_tt_ranks) - self.assertEqual(False, actual) - if not desired: - with self.assertRaises(ValueError): - tensor_train.TensorTrain((a, b, c), (10, 1, 2), claimed_tt_ranks) - - # Make dtypes inconsistent. - b_new = tf.cast(b, tf.float64) - actual = tensor_train._are_tt_cores_valid((a, b_new, c), (10, 1, 2), - claimed_tt_ranks) - self.assertEqual(False, actual) - with self.assertRaises(ValueError): - tensor_train.TensorTrain((a, b_new, c), (10, 1, 2), claimed_tt_ranks) - - def testTensorIndexing(self): - tens = initializers.random_tensor((3, 3, 4)) - with self.test_session() as sess: - desired = ops.full(tens)[:, :, :] - actual = ops.full(tens[:, :, :]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[1, :, :] - actual = ops.full(tens[1, :, :]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[1:2, 1, :] - actual = ops.full(tens[1:2, 1, :]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[0:3, :, 3] - actual = ops.full(tens[0:3, :, 3]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - desired = ops.full(tens)[1, :, 3] - actual = ops.full(tens[1, :, 3]) - desired, actual = sess.run([desired, actual]) - self.assertAllClose(desired, actual) - - # Wrong number of dims. - with self.assertRaises(ValueError): - tens[1, :, 3, :] - with self.assertRaises(ValueError): - tens[1, 1] - - def testPlaceholderTensorIndexing(self): - tens = initializers.random_tensor((3, 3, 4)) - with self.test_session() as sess: - start = tf.placeholder(tf.int32) - end = tf.placeholder(tf.int32) - desired = ops.full(tens)[1:3, 1, :3] - actual = ops.full(tens[start:end, start, :end]) - desired, actual = sess.run([desired, actual], {start: 1, end: 3}) - self.assertAllClose(desired, actual) - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/utils.py b/build/lib/t3f/utils.py deleted file mode 100644 index 7d26ab0d..00000000 --- a/build/lib/t3f/utils.py +++ /dev/null @@ -1,39 +0,0 @@ -import numpy as np -import tensorflow as tf - - -# TODO: substitute with native implementation when it's ready. -# https://github.com/tensorflow/tensorflow/issues/2075 -def unravel_index(indices, shape): - with tf.name_scope('unravel_index'): - indices = tf.expand_dims(indices, 0) - shape = tf.expand_dims(shape, 1) - strides_shifted = tf.cumprod(shape, exclusive=True, reverse=True) - res = (indices // strides_shifted) % shape - return tf.transpose(res, (1, 0)) - - -# TODO: get rid of this when TF fixes the NaN bugs in tf.svd: -# https://github.com/tensorflow/tensorflow/issues/8905 -def replace_tf_svd_with_np_svd(): - """Replaces tf.svd with np.svd. Slow, but a workaround for tf.svd bugs.""" - if hasattr(tf, 'original_svd'): - # This function has been already called and tf.svd is already replaced. - return - tf.original_svd = tf.svd - - def my_svd(tensor, full_matrices=False, compute_uv=True): - dtype = tensor.dtype - u, s, v = tf.py_func(np.linalg.svd, [tensor, full_matrices, compute_uv], - [dtype, dtype, dtype]) - s_, u_, v_ = tf.original_svd(tensor, full_matrices, compute_uv) - s = tf.reshape(s, s_.get_shape()) - u = tf.reshape(u, u_.get_shape()) - v = tf.reshape(v, v_.get_shape()) - # Converting numpy order of v dims to TF order. - order = range(tensor.get_shape().ndims) - order[-2], order[-1] = order[-1], order[-2] - v = tf.transpose(v, order) - return s, u, v - - tf.svd = my_svd diff --git a/build/lib/t3f/utils_test.py b/build/lib/t3f/utils_test.py deleted file mode 100644 index 170ab420..00000000 --- a/build/lib/t3f/utils_test.py +++ /dev/null @@ -1,26 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f import utils - - -class UtilsTest(tf.test.TestCase): - - def testUnravelIndex(self): - with self.test_session(): - # 2D. - shape = (7, 6) - linear_idx = [22, 41, 37] - desired = [[3, 4], [6, 5], [6, 1]] - actual = utils.unravel_index(linear_idx, shape) - self.assertAllEqual(desired, actual.eval()) - # 3D. - shape = (2, 3, 4) - linear_idx = [19, 17, 0, 23] - desired = [[1, 1, 3], [1, 1, 1], [0, 0, 0], [1, 2, 3]] - actual = utils.unravel_index(linear_idx, shape) - self.assertAllEqual(desired, actual.eval()) - - -if __name__ == "__main__": - tf.test.main() diff --git a/build/lib/t3f/variables.py b/build/lib/t3f/variables.py deleted file mode 100644 index 3c9cf569..00000000 --- a/build/lib/t3f/variables.py +++ /dev/null @@ -1,141 +0,0 @@ -import tensorflow as tf - -from t3f.tensor_train import TensorTrain -from t3f.tensor_train_batch import TensorTrainBatch - -eager_mode_supported = False -try: - from tensorflow.python.eager import context - eager_mode_supported = True -except ImportError: - pass - - -def get_variable(name, - dtype=None, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - validate_shape=True): - """Returns TensorTrain object with tf.Variables as the TT-cores. - - Args: - name: The name of the new or existing TensorTrain variable. - Used to name the TT-cores. - dtype: Type of the new or existing TensorTrain variable TT-cores (defaults - to DT_FLOAT). - initializer: TensorTrain or TensorTrainBatch, initializer for the variable - if one is created. - regularizer: A (TensorTrain -> Tensor or None) function; the result of - applying it on a newly created variable will be added to the collection - GraphKeys.REGULARIZATION_LOSSES and can be used for regularization. - trainable: If True also add the variable to the graph collection - GraphKeys.TRAINABLE_VARIABLES (see tf.Variable). - collections: List of graph collections keys to add the Variables - (underlying TT-cores). Defaults to [GraphKeys.GLOBAL_VARIABLES] - (see tf.Variable). - caching_device: Optional device string or function describing where - the Variable should be cached for reading. Defaults to the Variable's - device. If not None, caches on another device. Typical use is to cache - on the device where the Ops using the Variable reside, to deduplicate - copying through Switch and other conditional statements. - validate_shape: If False, allows the variable to be initialized with a value - of unknown shape. If True, the default, the shape of initial_value must be - known. - - Returns: - The created or existing `TensorTrain` object with tf.Variables TT-cores. - - Raises: - `ValueError`: when creating a new variable and shape is not declared, when - violating reuse during variable creation, or when initializer dtype and - dtype don't match. Reuse is set inside variable_scope. - """ - # TODO: support validate shape: check that the tensor dimensions are correct, - # but ignore the ranks. - # TODO: add validate ranks flag. - - reuse = tf.get_variable_scope().reuse - if not reuse and initializer is None: - raise ValueError('Scope reuse is False and initializer is not provided.') - - variable_cores = [] - in_eager_mode = False - if eager_mode_supported: - if context.in_eager_mode(): - in_eager_mode = True - - if reuse and not in_eager_mode: - # Find an existing variable in the collection. - path = tf.get_variable_scope().name - if path != '' and path[-1] != '/': - path += '/' - path += name - - found_v = None - for v in tf.get_collection('TensorTrainVariables'): - if v.name == path: - found_v = v - break - if found_v is None: - raise ValueError('ValueError: Variable %s does not exist, or was not ' - 'created with t3f.get_tt_variable(). Did you mean to ' - 'set reuse=None in VarScope?' % name) - with tf.variable_scope(name): - # Try to get the first core through tf.get_variable to check that we don't - # violate reuse: it will raise a ValueError otherwise. - tf.get_variable('core_0') - return found_v - else: - # Create new variable. - with tf.variable_scope(name): - num_dims = initializer.ndims() - for i in range(num_dims): - curr_core_var = tf.get_variable('core_%d' % i, - initializer=initializer.tt_cores[i], - dtype=dtype, trainable=trainable, - collections=collections, - caching_device=caching_device) - variable_cores.append(curr_core_var) - if isinstance(initializer, TensorTrain): - v = TensorTrain(variable_cores, initializer.get_raw_shape(), - initializer.get_tt_ranks(), - convert_to_tensors=False) - else: - v = TensorTrainBatch(variable_cores, initializer.get_raw_shape(), - initializer.get_tt_ranks(), initializer.batch_size, - convert_to_tensors=False) - - # Add the create TensorTrain object into a collection so that we can - # retrieve it in the future by get_tt_variable('name'). - tf.add_to_collection('TensorTrainVariables', v) - - # Run the regularizer if requested and save the resulting loss. - if regularizer: - with tf.name_scope(name + "/Regularizer/"): - loss = regularizer(v) - if loss is not None: - tf.logging.vlog(1, "Applied regularizer to %s and added the result %s " - "to REGULARIZATION_LOSSES.", v.name, loss.name) - tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, loss) - return v - - -def assign(ref, value, validate_shape=None, use_locking=None, name=None): - new_cores = [] - if name is None: - name = '' - with tf.variable_scope(name): - for i in range(ref.ndims()): - new_cores.append(tf.assign(ref.tt_cores[i], value.tt_cores[i], - use_locking=use_locking)) - if isinstance(value, TensorTrainBatch): - return TensorTrainBatch(new_cores, value.get_raw_shape(), - value.get_tt_ranks(), value.batch_size, - convert_to_tensors=False) - else: - return TensorTrain(new_cores, value.get_raw_shape(), - value.get_tt_ranks(), - convert_to_tensors=False) diff --git a/build/lib/t3f/variables_test.py b/build/lib/t3f/variables_test.py deleted file mode 100644 index de96a22d..00000000 --- a/build/lib/t3f/variables_test.py +++ /dev/null @@ -1,82 +0,0 @@ -import numpy as np -import tensorflow as tf - -from t3f import variables -from t3f import ops -from t3f import initializers - -class VariablesTest(tf.test.TestCase): - - def testGetExistingVariable(self): - init = initializers.random_tensor([2, 3, 2], tt_rank=2) - tt_1 = variables.get_variable('tt_1', initializer=init) - with tf.variable_scope('test'): - tt_2 = variables.get_variable('tt_2', initializer=init) - with self.test_session(): - tf.global_variables_initializer().run() - with self.assertRaises(ValueError): - # The variable already exists and scope.reuse is False by default. - variables.get_variable('tt_1') - with self.assertRaises(ValueError): - with tf.variable_scope('', reuse=True): - # The variable doesn't exist. - variables.get_variable('tt_3') - - with tf.variable_scope('', reuse=True): - tt_1_copy = variables.get_variable('tt_1') - self.assertAllClose(ops.full(tt_1).eval(), ops.full(tt_1_copy).eval()) - - with tf.variable_scope('', reuse=True): - # Again try to retrieve an existing variable, but pass an initializer - # and check that it still works. - tt_1_copy = variables.get_variable('tt_1', initializer=0 * init) - self.assertAllClose(ops.full(tt_1).eval(), ops.full(tt_1_copy).eval()) - - with self.assertRaises(ValueError): - with tf.variable_scope('', reuse=True): - # The variable is defined in a different scope - variables.get_variable('tt_2') - - with self.assertRaises(ValueError): - with tf.variable_scope('nottest', reuse=True): - # The variable is defined in a different scope - variables.get_variable('tt_2') - - with tf.variable_scope('test', reuse=True): - tt_2_copy = variables.get_variable('tt_2') - self.assertAllClose(ops.full(tt_2).eval(), ops.full(tt_2_copy).eval()) - - def testAttributes(self): - # Test that after converting an initializer into a variable all the - # attributes stays the same. - tens = initializers.random_tensor([2, 3, 2], tt_rank=2) - tens_v = variables.get_variable('tt_tens', initializer=tens) - mat = initializers.random_matrix([[3, 2, 2], [3, 3, 3]], tt_rank=3) - mat_v = variables.get_variable('tt_mat', initializer=mat) - for (init, var) in [[tens, tens_v], [mat, mat_v]]: - self.assertEqual(init.get_shape(), var.get_shape()) - self.assertEqual(init.get_raw_shape(), var.get_raw_shape()) - self.assertEqual(init.ndims(), var.ndims()) - self.assertEqual(init.get_tt_ranks(), var.get_tt_ranks()) - self.assertEqual(init.is_tt_matrix(), var.is_tt_matrix()) - - def testAssign(self): - old_init = initializers.random_tensor([2, 3, 2], tt_rank=2) - tt = variables.get_variable('tt', initializer=old_init) - new_init = initializers.random_tensor([2, 3, 2], tt_rank=2) - assigner = variables.assign(tt, new_init) - with self.test_session(): - tf.global_variables_initializer().run() - init_value = ops.full(tt).eval() - assigner_value = ops.full(assigner).eval() - after_value = ops.full(tt) - after_value = after_value.eval() - self.assertAllClose(assigner_value, after_value) - # Assert that the value actually changed: - abs_diff = np.linalg.norm((init_value - after_value).flatten()) - rel_diff = abs_diff / np.linalg.norm((init_value).flatten()) - self.assertGreater(rel_diff, 0.2) - - -if __name__ == "__main__": - tf.test.main() From ce686556f161c766a86573262b625a16d64e186c Mon Sep 17 00:00:00 2001 From: geom-score Date: Mon, 5 Mar 2018 19:53:48 +0300 Subject: [PATCH 006/233] moved to utils --- t3f/utils.py | 9 +++++++++ t3f/variables.py | 14 ++------------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/t3f/utils.py b/t3f/utils.py index 7d26ab0d..3649fcfa 100644 --- a/t3f/utils.py +++ b/t3f/utils.py @@ -37,3 +37,12 @@ def my_svd(tensor, full_matrices=False, compute_uv=True): return s, u, v tf.svd = my_svd + + +def in_eager_mode(): + """Checks whether tensorflow eager mode as avaialable and active.""" + try: + from tensorflow.python.eager import context + return context.in_eager_mode() + except ImportError: + return False diff --git a/t3f/variables.py b/t3f/variables.py index 3c9cf569..7e2422a5 100644 --- a/t3f/variables.py +++ b/t3f/variables.py @@ -2,13 +2,7 @@ from t3f.tensor_train import TensorTrain from t3f.tensor_train_batch import TensorTrainBatch - -eager_mode_supported = False -try: - from tensorflow.python.eager import context - eager_mode_supported = True -except ImportError: - pass +from t3f import utils def get_variable(name, @@ -62,12 +56,8 @@ def get_variable(name, raise ValueError('Scope reuse is False and initializer is not provided.') variable_cores = [] - in_eager_mode = False - if eager_mode_supported: - if context.in_eager_mode(): - in_eager_mode = True - if reuse and not in_eager_mode: + if reuse and not utils.in_eager_mode(): # Find an existing variable in the collection. path = tf.get_variable_scope().name if path != '' and path[-1] != '/': From a4727f1e6ac15efaa9659494ff2488878c4a788e Mon Sep 17 00:00:00 2001 From: Valentin Khrulkov Date: Wed, 7 Mar 2018 11:02:40 +0300 Subject: [PATCH 007/233] Update utils.py --- t3f/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/utils.py b/t3f/utils.py index 3649fcfa..7f828538 100644 --- a/t3f/utils.py +++ b/t3f/utils.py @@ -40,7 +40,7 @@ def my_svd(tensor, full_matrices=False, compute_uv=True): def in_eager_mode(): - """Checks whether tensorflow eager mode as avaialable and active.""" + """Checks whether tensorflow eager mode is avaialable and active.""" try: from tensorflow.python.eager import context return context.in_eager_mode() From 460599e2cd8a545930dac9783bd35e3725d9e4aa Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 20 Mar 2018 13:30:00 +0300 Subject: [PATCH 008/233] Add new tf versions --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index d7284bea..f44bdc66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,9 @@ env: - TF_VERSION=1.2 - TF_VERSION=1.3 - TF_VERSION=1.4 + - TF_VERSION=1.5 + - TF_VERSION=1.6 + - TF_VERSION=1.7 # command to install dependencies install: # Python 2.7 has very old pip by default that doesn't have tensorflow. From f9abd539c96544aadb8894b1153d40b5ee367587 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 20 Mar 2018 13:30:33 +0300 Subject: [PATCH 009/233] 1.7 is not out yet --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f44bdc66..7089e7ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ env: - TF_VERSION=1.4 - TF_VERSION=1.5 - TF_VERSION=1.6 - - TF_VERSION=1.7 # command to install dependencies install: # Python 2.7 has very old pip by default that doesn't have tensorflow. From d670e1784784af7844aca938feb68056a53374eb Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Fri, 6 Apr 2018 17:06:30 +0300 Subject: [PATCH 010/233] Add approximate module to the docs --- docs/index.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4f6c21f6..8ec97263 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. Welcome to t3f documentation! -=============================== +============================= Module contents --------------- @@ -24,10 +24,17 @@ t3f\.utils module :show-inheritance: t3f\.kronecker module ------------------ +--------------------- .. automodule:: t3f.kronecker :members: :undoc-members: :show-inheritance: +t3f\.approximate module +----------------------- + +.. automodule:: t3f.approximate + :members: + :undoc-members: + :show-inheritance: From d74afa6a743e0dcd9497a2d900906875f62925b5 Mon Sep 17 00:00:00 2001 From: Valentin Khrulkov Date: Fri, 6 Apr 2018 17:47:49 +0300 Subject: [PATCH 011/233] Update tensor_train_base.py --- t3f/tensor_train_base.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/t3f/tensor_train_base.py b/t3f/tensor_train_base.py index 7bad0a07..864483b6 100644 --- a/t3f/tensor_train_base.py +++ b/t3f/tensor_train_base.py @@ -5,19 +5,6 @@ # TODO: check the methods of _TensorLike class TensorTrainBase(object): """An abstract class that represents a collection of Tensor Train cores. - ``` - @@__init__ - @@get_raw_shape - @@get_shape - @@tt_cores - @@dtype - @@name - @@graph - @@ndims - @@get_tt_ranks - @@is_tt_matrix - @@is_variable - @@eval """ def __init__(self, tt_cores): From 8e1b6a6146272fbae633902eaf037297060cb0dc Mon Sep 17 00:00:00 2001 From: Valentin Khrulkov Date: Fri, 6 Apr 2018 17:48:41 +0300 Subject: [PATCH 012/233] Update tensor_train.py --- t3f/tensor_train.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/t3f/tensor_train.py b/t3f/tensor_train.py index b884e92f..cd96920c 100644 --- a/t3f/tensor_train.py +++ b/t3f/tensor_train.py @@ -8,21 +8,6 @@ class TensorTrain(TensorTrainBase): """Represents a Tensor Train object (a TT-tensor or TT-matrix). t3f represents a Tensor Train object as a tuple of TT-cores. - ``` - @@__init__ - @@get_raw_shape - @@get_shape - @@tt_cores - @@dtype - @@name - @@graph - @@ndims - @@get_tt_ranks - @@left_tt_rank_dim - @@right_tt_rank_dim - @@is_tt_matrix - @@is_variable - @@eval """ def __init__(self, tt_cores, shape=None, tt_ranks=None, convert_to_tensors=True): From 93a75ea26bbc9daccfba2e6e191e506c765f7ab9 Mon Sep 17 00:00:00 2001 From: Valentin Khrulkov Date: Fri, 6 Apr 2018 17:49:33 +0300 Subject: [PATCH 013/233] Update tensor_train_batch.py --- t3f/tensor_train_batch.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/t3f/tensor_train_batch.py b/t3f/tensor_train_batch.py index 5ff0410b..49bc1ee6 100644 --- a/t3f/tensor_train_batch.py +++ b/t3f/tensor_train_batch.py @@ -10,21 +10,6 @@ class TensorTrainBatch(TensorTrainBase): """Represents a batch of Tensor Train objects (TT-tensors or TT-matrices). t3f represents a Tensor Train object as a tuple of TT-cores. - ``` - @@__init__ - @@get_raw_shape - @@get_shape - @@tt_cores - @@dtype - @@name - @@graph - @@ndims - @@get_tt_ranks - @@left_tt_rank_dim - @@right_tt_rank_dim - @@is_tt_matrix - @@is_variable - @@eval """ def __init__(self, tt_cores, shape=None, tt_ranks=None, batch_size=None, From 777adcefc188e0a86319ea8dca126d4ab316c58a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 8 Apr 2018 08:17:44 +0200 Subject: [PATCH 014/233] Use tf 1.5 for documentation build (according to https://github.com/tensorflow/tensorflow/issues/17411 the problem with documentation build maybe because of newer tf) --- docs/tf_requirement.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tf_requirement.txt b/docs/tf_requirement.txt index 5cd0635d..8a20c945 100644 --- a/docs/tf_requirement.txt +++ b/docs/tf_requirement.txt @@ -4,4 +4,4 @@ # the docs with readthedocs.org: they fetch a fresh version of the # library on each build and it doesn't import properly without tensorflow # being installed. -tensorflow>=1.0 \ No newline at end of file +tensorflow>=1.0,<=1.5 From a5fc6e5d688d2132dda4f084f22ade95ba7120d6 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 22 Apr 2018 20:10:02 +0300 Subject: [PATCH 015/233] Init integer overflow tests --- t3f/tensor_train_batch_test.py | 9 +++++++++ t3f/tensor_train_test.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/t3f/tensor_train_batch_test.py b/t3f/tensor_train_batch_test.py index b609f1a7..2538b2b3 100644 --- a/t3f/tensor_train_batch_test.py +++ b/t3f/tensor_train_batch_test.py @@ -72,6 +72,15 @@ def testPlaceholderTensorIndexing(self): desired, actual = sess.run([desired, actual], {start: 1, end: 3}) self.assertAllClose(desired, actual) + def testShapeOverflow(self): + large_shape = [10] * 20 + tensor = initializers.random_matrix_batch([large_shape, large_shape], batch_size=5) + try: + shape = tensor.get_shape() + self.assertEqual([5, 10 ** 20, 10 ** 20], shape) + except ValueError: + self.fail('Integer overflow') + if __name__ == "__main__": tf.test.main() diff --git a/t3f/tensor_train_test.py b/t3f/tensor_train_test.py index 22983424..0a57663c 100644 --- a/t3f/tensor_train_test.py +++ b/t3f/tensor_train_test.py @@ -134,5 +134,14 @@ def testPlaceholderTensorIndexing(self): desired, actual = sess.run([desired, actual], {start: 1, end: 3}) self.assertAllClose(desired, actual) + def testShapeOverflow(self): + large_shape = [10] * 20 + matrix = initializers.matrix_zeros([large_shape, large_shape]) + try: + shape = matrix.get_shape() + self.assertEqual([10 ** 20, 10 ** 20], shape) + except ValueError: + self.fail('Integer overflow') + if __name__ == "__main__": tf.test.main() From 6cefce8de62af7f3115310684144399a690c6923 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 22 Apr 2018 20:11:49 +0300 Subject: [PATCH 016/233] Init solution to the problem --- t3f/tensor_train_base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/t3f/tensor_train_base.py b/t3f/tensor_train_base.py index 864483b6..f4c25b79 100644 --- a/t3f/tensor_train_base.py +++ b/t3f/tensor_train_base.py @@ -1,3 +1,4 @@ +from functools import reduce import numpy as np import tensorflow as tf @@ -30,9 +31,11 @@ def get_shape(self): """ raw_shape = self.get_raw_shape() if self.is_tt_matrix(): + # Use python prod instead of np.prod to avoid overflows. + prod_f = lambda arr: reduce(lambda x, y: x*y, arr) # TODO: as list is not available if shape is partly known. - m = np.prod(raw_shape[0].as_list()) - n = np.prod(raw_shape[1].as_list()) + m = prod_f(raw_shape[0].as_list()) + n = prod_f(raw_shape[1].as_list()) return tf.TensorShape((m, n)) else: return raw_shape[0] From 42c2baffce6ee9903f5e9499a84d06cd8c43f453 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 22 Apr 2018 20:41:15 +0300 Subject: [PATCH 017/233] Simplify tests --- t3f/tensor_train_batch_test.py | 7 ++----- t3f/tensor_train_test.py | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/t3f/tensor_train_batch_test.py b/t3f/tensor_train_batch_test.py index 2538b2b3..41894f1e 100644 --- a/t3f/tensor_train_batch_test.py +++ b/t3f/tensor_train_batch_test.py @@ -75,11 +75,8 @@ def testPlaceholderTensorIndexing(self): def testShapeOverflow(self): large_shape = [10] * 20 tensor = initializers.random_matrix_batch([large_shape, large_shape], batch_size=5) - try: - shape = tensor.get_shape() - self.assertEqual([5, 10 ** 20, 10 ** 20], shape) - except ValueError: - self.fail('Integer overflow') + shape = tensor.get_shape() + self.assertEqual([5, 10 ** 20, 10 ** 20], shape) if __name__ == "__main__": diff --git a/t3f/tensor_train_test.py b/t3f/tensor_train_test.py index 0a57663c..ec99dd7d 100644 --- a/t3f/tensor_train_test.py +++ b/t3f/tensor_train_test.py @@ -137,11 +137,8 @@ def testPlaceholderTensorIndexing(self): def testShapeOverflow(self): large_shape = [10] * 20 matrix = initializers.matrix_zeros([large_shape, large_shape]) - try: - shape = matrix.get_shape() - self.assertEqual([10 ** 20, 10 ** 20], shape) - except ValueError: - self.fail('Integer overflow') + shape = matrix.get_shape() + self.assertEqual([10 ** 20, 10 ** 20], shape) if __name__ == "__main__": tf.test.main() From f9b4544b359c48be3bcae9fd2d6e3a99c8a18e44 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 22 Apr 2018 20:45:21 +0300 Subject: [PATCH 018/233] Test lazy shape as well --- t3f/shapes_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 t3f/shapes_test.py diff --git a/t3f/shapes_test.py b/t3f/shapes_test.py new file mode 100644 index 00000000..bf0decaf --- /dev/null +++ b/t3f/shapes_test.py @@ -0,0 +1,16 @@ +import tensorflow as tf + +from t3f import initializers +from t3f import shapes + + +class ShapesTest(tf.test.TestCase): + + def testLazyShapeOverflow(self): + large_shape = [10] * 20 + tensor = initializers.random_matrix_batch([large_shape, large_shape], batch_size=5) + self.assertAllEqual([5, 10 ** 20, 10 ** 20], shapes.lazy_shape(tensor)) + + +if __name__ == "__main__": + tf.test.main() From 684e391abe26c7ba6a8803dc061a008fa063f2a5 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 22 Apr 2018 21:20:10 +0300 Subject: [PATCH 019/233] Add tf1.7 back (its out) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7089e7ca..f44bdc66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - TF_VERSION=1.4 - TF_VERSION=1.5 - TF_VERSION=1.6 + - TF_VERSION=1.7 # command to install dependencies install: # Python 2.7 has very old pip by default that doesn't have tensorflow. From 8de05e98eace45ebf340a067f855111ce31fd5c9 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 22 Apr 2018 21:40:51 +0300 Subject: [PATCH 020/233] Avoid testing TF 1.5 because it conflicts with python 3.6 (the problem is on the TF side). It should work nevertheless. https://github.com/tensorflow/tensorflow/issues/16488 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f44bdc66..6f27b5df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ env: - TF_VERSION=1.2 - TF_VERSION=1.3 - TF_VERSION=1.4 - - TF_VERSION=1.5 - TF_VERSION=1.6 - TF_VERSION=1.7 # command to install dependencies From a074822d9285d171c425827a9a04c0f632d11a27 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 22 Apr 2018 21:41:43 +0300 Subject: [PATCH 021/233] Add new tested TF versions into the Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e04e8539..8648e869 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ TensorFlow implementation of the Tensor Train (TT) -Toolbox. API is available via [readthedocs](https://t3f.readthedocs.io/en/latest/). # Installation -T3f assumes you have a working TensorFlow installation, supported versions are from 1.0 to 1.4 (see [here](https://www.tensorflow.org/versions/r1.4/install/) for TF 1.4 installation instructions). +T3f assumes you have a working TensorFlow installation, supported versions are from 1.0 to 1.7 (see [here](https://www.tensorflow.org/versions/r1.7/install/) for TF 1.7 installation instructions). We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. Then simply run ```bash From 9e47c7862389d8592bc0ddd98c3786a2e12f28e4 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 22 Apr 2018 21:47:12 +0300 Subject: [PATCH 022/233] Add tested python versions into readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8648e869..2d51fbde 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ API is available via [readthedocs](https://t3f.readthedocs.io/en/latest/). # Installation T3f assumes you have a working TensorFlow installation, supported versions are from 1.0 to 1.7 (see [here](https://www.tensorflow.org/versions/r1.7/install/) for TF 1.7 installation instructions). -We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. +We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. We test with Python 2.7, 3.4, 3.5, 3.6. Then simply run ```bash pip install t3f From 6fd3a666ed9df6c24a215a85a1465d19a3b8f737 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 22 Apr 2018 21:48:37 +0300 Subject: [PATCH 023/233] Better python version description --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2d51fbde..35ef70b4 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ TensorFlow implementation of the Tensor Train (TT) -Toolbox. API is available via [readthedocs](https://t3f.readthedocs.io/en/latest/). # Installation -T3f assumes you have a working TensorFlow installation, supported versions are from 1.0 to 1.7 (see [here](https://www.tensorflow.org/versions/r1.7/install/) for TF 1.7 installation instructions). -We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. We test with Python 2.7, 3.4, 3.5, 3.6. +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, supported versions are from 1.0 to 1.7 (see [here](https://www.tensorflow.org/versions/r1.7/install/) for TF 1.7 installation instructions). +We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. Then simply run ```bash pip install t3f From cb84feb8fab272a03f6782ac643c3200869eb3c6 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Thu, 3 May 2018 13:26:52 +0300 Subject: [PATCH 024/233] Fix name_scope bug (fails only in eager for some reason) --- t3f/tensor_train.py | 2 +- t3f/tensor_train_batch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/tensor_train.py b/t3f/tensor_train.py index cd96920c..ba59406a 100644 --- a/t3f/tensor_train.py +++ b/t3f/tensor_train.py @@ -36,7 +36,7 @@ def __init__(self, tt_cores, shape=None, tt_ranks=None, convert_to_tensors=True) tt_cores = list(tt_cores) if convert_to_tensors: # TODO: what does this namescope do? - with tf.name_scope("TensorTrain", tt_cores): + with tf.name_scope("TensorTrain", values=tt_cores): for i in range(len(tt_cores)): name = "core%d" % i tt_cores[i] = tf.convert_to_tensor(tt_cores[i], name=name) diff --git a/t3f/tensor_train_batch.py b/t3f/tensor_train_batch.py index 49bc1ee6..b521c33c 100644 --- a/t3f/tensor_train_batch.py +++ b/t3f/tensor_train_batch.py @@ -42,7 +42,7 @@ def __init__(self, tt_cores, shape=None, tt_ranks=None, batch_size=None, tt_cores = list(tt_cores) if convert_to_tensors: # TODO: what does this namescope do? - with tf.name_scope("TensorTrainBatch", tt_cores): + with tf.name_scope("TensorTrainBatch", values=tt_cores): for i in range(len(tt_cores)): name = "core%d" % i tt_cores[i] = tf.convert_to_tensor(tt_cores[i], name=name) From eee81148898e86449a40f5c8b91dace2d8ca22aa Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sat, 5 May 2018 20:08:53 +0300 Subject: [PATCH 025/233] Init name scopes for ops --- t3f/ops.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index e57fd9ab..8bb4040f 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -11,21 +11,23 @@ # TODO: add complexities to the comments. -def full(tt): +def full(tt, name='tt_to_full'): """Converts a TensorTrain into a regular tensor or matrix (tf.Tensor). Args: tt: `TensorTrain` or `TensorTrainBatch` object. + name: string, name of the Op. Returns: tf.Tensor. """ - if isinstance(tt, TensorTrainBatch): - # Batch of Tensor Trains. - return _full_tt_batch(tt) - else: - # TensorTrain object (not batch). - return _full_tt(tt) + with tf.name_scope(name, tt.tt_cores): + if isinstance(tt, TensorTrainBatch): + # Batch of Tensor Trains. + return _full_tt_batch(tt) + else: + # TensorTrain object (not batch). + return _full_tt(tt) def _full_tt(tt): From 95e4152fe0c7a0d7961f09c469258cb9196e0dad Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sat, 5 May 2018 20:36:43 +0300 Subject: [PATCH 026/233] Add name_scope too all ops in ops.py --- t3f/ops.py | 672 ++++++++++++++++++++++++++++------------------------- 1 file changed, 354 insertions(+), 318 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index 8bb4040f..a3f10e18 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -275,7 +275,7 @@ def tt_sparse_matmul(tt_matrix_a, sparse_matrix_b): raise NotImplementedError -def matmul(a, b): +def matmul(a, b, name='matmul'): """Multiplies two matrices that can be TT-, dense, or sparse. Note that multiplication of two TT-matrices returns a TT-matrix with much @@ -288,6 +288,7 @@ def matmul(a, b): size M x N b: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor of size N x P + name: string, name of the Op. Returns If both arguments are `TensorTrain` objects, returns a `TensorTrain` @@ -299,15 +300,20 @@ def matmul(a, b): """ # TODO: is it safe to check types? What if a class is derived from TT? if isinstance(a, TensorTrainBase) and isinstance(b, TensorTrainBase): - return tt_tt_matmul(a, b) + with tf.name_scope(name, a.tt_cores + b.tt_cores): + return tt_tt_matmul(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): - return tt_dense_matmul(a, b) + with tf.name_scope(name, a.tt_cores + [b]): + return tt_dense_matmul(a, b) elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): - return dense_tt_matmul(a, b) + with tf.name_scope(name, [a] + b.tt_cores): + return dense_tt_matmul(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): - return tt_sparse_matmul(a, b) + with tf.name_scope(name, a.tt_cores + [b]): + return tt_sparse_matmul(a, b) elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): - return sparse_tt_matmul(a, b) + with tf.name_scope(name, [a] + b.tt_cores): + return sparse_tt_matmul(a, b) else: raise ValueError('Argument types are not supported in matmul: %s x %s' % (a, b)) @@ -497,7 +503,7 @@ def sparse_tt_flat_inner(sparse_a, tt_b): raise NotImplementedError -def flat_inner(a, b): +def flat_inner(a, b, name='flat_inner'): """Inner product along all axis. The shapes of a and b should coincide. @@ -505,6 +511,7 @@ def flat_inner(a, b): Args: a: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor b: `TensorTrain`, `TensorTrainBatch`, tf.Tensor, or tf.SparseTensor + name: string, name of the Op. Returns a number @@ -515,15 +522,20 @@ def flat_inner(a, b): """ # TODO: is it safe to check types? What if a class is derived from TT? if isinstance(a, TensorTrainBase) and isinstance(b, TensorTrainBase): - return tt_tt_flat_inner(a, b) + with tf.name_scope(name, a.tt_cores + b.tt_cores): + return tt_tt_flat_inner(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): - return tt_dense_flat_inner(a, b) + with tf.name_scope(name, a.tt_cores + [b]): + return tt_dense_flat_inner(a, b) elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): - return dense_tt_flat_inner(a, b) + with tf.name_scope(name, [a] + b.tt_cores): + return dense_tt_flat_inner(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): - return tt_sparse_flat_inner(a, b) + with tf.name_scope(name, a.tt_cores + [b]): + return tt_sparse_flat_inner(a, b) elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): - return sparse_tt_flat_inner(a, b) + with tf.name_scope(name, [a] + b.tt_cores): + return sparse_tt_flat_inner(a, b) else: raise ValueError('Argument types are not supported in flat_inner: %s x %s' % (a, b)) @@ -673,7 +685,7 @@ def _add_batch_matrix_cores(tt_a, tt_b): return tt_cores, batch_size -def add(tt_a, tt_b): +def add(tt_a, tt_b, name='add'): """Returns a TensorTrain corresponding to elementwise sum tt_a + tt_b. The shapes of tt_a and tt_b should coincide. @@ -684,6 +696,7 @@ def add(tt_a, tt_b): Args: tt_a: `TensorTrain`, `TensorTrainBatch`, TT-tensor, or TT-matrix tt_b: `TensorTrain`, `TensorTrainBatch`, TT-tensor, or TT-matrix + name: string, name of the Op. Returns a `TensorTrain` object corresponding to the element-wise sum of arguments if @@ -706,33 +719,36 @@ def add(tt_a, tt_b): raise ValueError('The batch sizes are different and not 1, broadcasting is ' 'not available.') - is_batch_case = isinstance(tt_a, TensorTrainBatch) or isinstance(tt_b, TensorTrainBatch) - batch_size = None - if is_batch_case: - if tt_a.is_tt_matrix(): - tt_cores, batch_size = _add_batch_matrix_cores(tt_a, tt_b) + with tf.name_scope(name, values=tt_a.tt_cores + tt_b.tt_cores): + is_a_batch = isinstance(tt_a, TensorTrainBatch) + is_b_batch = isinstance(tt_b, TensorTrainBatch) + is_batch_case = is_a_batch or is_b_batch + batch_size = None + if is_batch_case: + if tt_a.is_tt_matrix(): + tt_cores, batch_size = _add_batch_matrix_cores(tt_a, tt_b) + else: + tt_cores, batch_size = _add_batch_tensor_cores(tt_a, tt_b) else: - tt_cores, batch_size = _add_batch_tensor_cores(tt_a, tt_b) - else: - if tt_a.is_tt_matrix(): - tt_cores = _add_matrix_cores(tt_a, tt_b) + if tt_a.is_tt_matrix(): + tt_cores = _add_matrix_cores(tt_a, tt_b) + else: + tt_cores = _add_tensor_cores(tt_a, tt_b) + + out_ranks = [1] + static_a_ranks = tt_a.get_tt_ranks() + static_b_ranks = tt_b.get_tt_ranks() + for core_idx in range(1, ndims): + out_ranks.append(static_a_ranks[core_idx] + static_b_ranks[core_idx]) + out_ranks.append(1) + if is_batch_case: + return TensorTrainBatch(tt_cores, tt_a.get_raw_shape(), out_ranks, + batch_size) else: - tt_cores = _add_tensor_cores(tt_a, tt_b) + return TensorTrain(tt_cores, tt_a.get_raw_shape(), out_ranks) - out_ranks = [1] - static_a_ranks = tt_a.get_tt_ranks() - static_b_ranks = tt_b.get_tt_ranks() - for core_idx in range(1, ndims): - out_ranks.append(static_a_ranks[core_idx] + static_b_ranks[core_idx]) - out_ranks.append(1) - if is_batch_case: - return TensorTrainBatch(tt_cores, tt_a.get_raw_shape(), out_ranks, - batch_size) - else: - return TensorTrain(tt_cores, tt_a.get_raw_shape(), out_ranks) - -def multiply(tt_left, right): +def multiply(tt_left, right, name='multiply'): """Returns a TensorTrain corresponding to element-wise product tt_left * right. Supports broadcasting: @@ -747,6 +763,7 @@ def multiply(tt_left, right): Args: tt_left: `TensorTrain` OR `TensorTrainBatch` right: `TensorTrain` OR `TensorTrainBatch` OR a number. + name: string, name of the Op. Returns a `TensorTrain` or `TensorTrainBatch` object corresponding to the @@ -756,153 +773,156 @@ def multiply(tt_left, right): ValueError if the arguments shapes do not coincide or broadcasting is not possible. """ - is_left_batch = isinstance(tt_left, TensorTrainBatch) is_right_batch = isinstance(right, TensorTrainBatch) is_batch_case = is_left_batch or is_right_batch ndims = tt_left.ndims() if not isinstance(right, TensorTrainBase): - # Assume right is a number, not TensorTrain. - # To squash right uniformly across TT-cores we pull its absolute value - # and raise to the power 1/ndims. First TT-core is multiplied by the sign - # of right. - tt_cores = list(tt_left.tt_cores) - fact = tf.pow(tf.cast(tf.abs(right), tt_left.dtype), 1.0 / ndims) - sign = tf.cast(tf.sign(right), tt_left.dtype) - for i in range(len(tt_cores)): - tt_cores[i] = fact * tt_cores[i] - - tt_cores[0] = tt_cores[0] * sign - out_ranks = tt_left.get_tt_ranks() - if is_left_batch: - out_batch_size = tt_left.batch_size + with tf.name_scope(name, values=tt_left.tt_cores+[right]): + # Assume right is a number, not TensorTrain. + # To squash right uniformly across TT-cores we pull its absolute value + # and raise to the power 1/ndims. First TT-core is multiplied by the sign + # of right. + tt_cores = list(tt_left.tt_cores) + fact = tf.pow(tf.cast(tf.abs(right), tt_left.dtype), 1.0 / ndims) + sign = tf.cast(tf.sign(right), tt_left.dtype) + for i in range(len(tt_cores)): + tt_cores[i] = fact * tt_cores[i] + + tt_cores[0] = tt_cores[0] * sign + out_ranks = tt_left.get_tt_ranks() + if is_left_batch: + out_batch_size = tt_left.batch_size else: + with tf.name_scope(name, values=tt_left.tt_cores+right.tt_cores): - if tt_left.is_tt_matrix() != right.is_tt_matrix(): - raise ValueError('The arguments should be both TT-tensors or both ' - 'TT-matrices') - - if tt_left.get_raw_shape() != right.get_raw_shape(): - raise ValueError('The arguments should have the same shape.') - - out_batch_size = 1 - dependencies = [] - can_determine_if_broadcast = True - if is_left_batch and is_right_batch: - if tt_left.batch_size is None and right.batch_size is None: - can_determine_if_broadcast = False - elif tt_left.batch_size is None and right.batch_size is not None: - if right.batch_size > 1: - can_determine_if_broadcast = False - elif tt_left.batch_size is not None and right.batch_size is None: - if tt_left.batch_size > 1: - can_determine_if_broadcast = False - - if not can_determine_if_broadcast: - # Cannot determine if broadcasting is needed. Avoid broadcasting and - # assume elementwise multiplication AND add execution time assert to print - # a better error message if the batch sizes turn out to be different. - - message = ('The batch sizes were unknown on compilation stage, so ' - 'assumed elementwise multiplication (i.e. no broadcasting). ' - 'Now it seems that they are different after all :') - - data = [message, shapes.lazy_batch_size(tt_left), ' x ', - shapes.lazy_batch_size(right)] - bs_eq = tf.assert_equal(shapes.lazy_batch_size(tt_left), - shapes.lazy_batch_size(right), data=data) - - dependencies.append(bs_eq) - - do_broadcast = shapes.is_batch_broadcasting_possible(tt_left, right) - if not can_determine_if_broadcast: - # Assume elementwise multiplication if broadcasting cannot be determined - # on compilation stage. - do_broadcast = False - if not do_broadcast and can_determine_if_broadcast: - raise ValueError('The batch sizes are different and not 1, broadcasting ' - 'is not available.') - - a_ranks = shapes.lazy_tt_ranks(tt_left) - b_ranks = shapes.lazy_tt_ranks(right) - shape = shapes.lazy_raw_shape(tt_left) - - output_str = '' - bs_str_left = '' - bs_str_right = '' + if tt_left.is_tt_matrix() != right.is_tt_matrix(): + raise ValueError('The arguments should be both TT-tensors or both ' + 'TT-matrices') - if is_batch_case: + if tt_left.get_raw_shape() != right.get_raw_shape(): + raise ValueError('The arguments should have the same shape.') + + out_batch_size = 1 + dependencies = [] + can_determine_if_broadcast = True if is_left_batch and is_right_batch: - # Both arguments are batches of equal size. - if tt_left.batch_size == right.batch_size or not can_determine_if_broadcast: - bs_str_left = 'n' - bs_str_right = 'n' - output_str = 'n' - if not can_determine_if_broadcast: - out_batch_size = None + if tt_left.batch_size is None and right.batch_size is None: + can_determine_if_broadcast = False + elif tt_left.batch_size is None and right.batch_size is not None: + if right.batch_size > 1: + can_determine_if_broadcast = False + elif tt_left.batch_size is not None and right.batch_size is None: + if tt_left.batch_size > 1: + can_determine_if_broadcast = False + + if not can_determine_if_broadcast: + # Cannot determine if broadcasting is needed. Avoid broadcasting and + # assume elementwise multiplication AND add execution time assert to + # print a better error message if the batch sizes turn out to be + # different. + + message = ('The batch sizes were unknown on compilation stage, so ' + 'assumed elementwise multiplication (i.e. no broadcasting). ' + 'Now it seems that they are different after all :') + + data = [message, shapes.lazy_batch_size(tt_left), ' x ', + shapes.lazy_batch_size(right)] + bs_eq = tf.assert_equal(shapes.lazy_batch_size(tt_left), + shapes.lazy_batch_size(right), data=data) + + dependencies.append(bs_eq) + + do_broadcast = shapes.is_batch_broadcasting_possible(tt_left, right) + if not can_determine_if_broadcast: + # Assume elementwise multiplication if broadcasting cannot be determined + # on compilation stage. + do_broadcast = False + if not do_broadcast and can_determine_if_broadcast: + raise ValueError('The batch sizes are different and not 1, broadcasting ' + 'is not available.') + + a_ranks = shapes.lazy_tt_ranks(tt_left) + b_ranks = shapes.lazy_tt_ranks(right) + shape = shapes.lazy_raw_shape(tt_left) + + output_str = '' + bs_str_left = '' + bs_str_right = '' + + if is_batch_case: + if is_left_batch and is_right_batch: + # Both arguments are batches of equal size. + if tt_left.batch_size == right.batch_size or not can_determine_if_broadcast: + bs_str_left = 'n' + bs_str_right = 'n' + output_str = 'n' + if not can_determine_if_broadcast: + out_batch_size = None + else: + out_batch_size = tt_left.batch_size else: - out_batch_size = tt_left.batch_size + # Broadcasting (e.g batch_sizes are 1 and n>1). + bs_str_left = 'n' + bs_str_right = 'm' + output_str = 'nm' + if tt_left.batch_size is None or tt_left.batch_size > 1: + out_batch_size = tt_left.batch_size + else: + out_batch_size = right.batch_size else: - # Broadcasting (e.g batch_sizes are 1 and n>1). - bs_str_left = 'n' - bs_str_right = 'm' - output_str = 'nm' - if tt_left.batch_size is None or tt_left.batch_size > 1: + # One of the arguments is TensorTrain. + if is_left_batch: + bs_str_left = 'n' + bs_str_right = '' out_batch_size = tt_left.batch_size else: + bs_str_left = '' + bs_str_right = 'n' out_batch_size = right.batch_size - else: - # One of the arguments is TensorTrain. - if is_left_batch: - bs_str_left = 'n' - bs_str_right = '' - out_batch_size = tt_left.batch_size + output_str = 'n' + + is_matrix = tt_left.is_tt_matrix() + tt_cores = [] + + for core_idx in range(ndims): + a_core = tt_left.tt_cores[core_idx] + b_core = right.tt_cores[core_idx] + left_rank = a_ranks[core_idx] * b_ranks[core_idx] + right_rank = a_ranks[core_idx + 1] * b_ranks[core_idx + 1] + if is_matrix: + with tf.control_dependencies(dependencies): + curr_core = tf.einsum('{0}aijb,{1}cijd->{2}acijbd'.format(bs_str_left, + bs_str_right, output_str), a_core, b_core) + curr_core = tf.reshape(curr_core, (-1, left_rank, + shape[0][core_idx], + shape[1][core_idx], + right_rank)) + if not is_batch_case: + curr_core = tf.squeeze(curr_core, axis=0) else: - bs_str_left = '' - bs_str_right = 'n' - out_batch_size = right.batch_size - output_str = 'n' - - is_matrix = tt_left.is_tt_matrix() - tt_cores = [] - - for core_idx in range(ndims): - a_core = tt_left.tt_cores[core_idx] - b_core = right.tt_cores[core_idx] - left_rank = a_ranks[core_idx] * b_ranks[core_idx] - right_rank = a_ranks[core_idx + 1] * b_ranks[core_idx + 1] - if is_matrix: - with tf.control_dependencies(dependencies): - curr_core = tf.einsum('{0}aijb,{1}cijd->{2}acijbd'.format(bs_str_left, - bs_str_right, output_str), a_core, b_core) - curr_core = tf.reshape(curr_core, (-1, left_rank, - shape[0][core_idx], - shape[1][core_idx], - right_rank)) - if not is_batch_case: + with tf.control_dependencies(dependencies): + curr_core = tf.einsum('{0}aib,{1}cid->{2}acibd'.format(bs_str_left, + bs_str_right, output_str), a_core, b_core) + curr_core = tf.reshape(curr_core, (-1, left_rank, + shape[0][core_idx], right_rank)) + if not is_batch_case: curr_core = tf.squeeze(curr_core, axis=0) - else: - with tf.control_dependencies(dependencies): - curr_core = tf.einsum('{0}aib,{1}cid->{2}acibd'.format(bs_str_left, - bs_str_right, output_str), a_core, b_core) - curr_core = tf.reshape(curr_core, (-1, left_rank, - shape[0][core_idx], right_rank)) - if not is_batch_case: - curr_core = tf.squeeze(curr_core, axis=0) - tt_cores.append(curr_core) + tt_cores.append(curr_core) - combined_ranks = zip(tt_left.get_tt_ranks(), right.get_tt_ranks()) - out_ranks = [a * b for a, b in combined_ranks] + combined_ranks = zip(tt_left.get_tt_ranks(), right.get_tt_ranks()) + out_ranks = [a * b for a, b in combined_ranks] - if not is_batch_case: - return TensorTrain(tt_cores, tt_left.get_raw_shape(), out_ranks) - else: - return TensorTrainBatch(tt_cores, tt_left.get_raw_shape(), out_ranks, - batch_size=out_batch_size) + if not is_batch_case: + return TensorTrain(tt_cores, tt_left.get_raw_shape(), out_ranks) + else: + return TensorTrainBatch(tt_cores, tt_left.get_raw_shape(), out_ranks, + batch_size=out_batch_size) -def frobenius_norm_squared(tt, differentiable=False): +def frobenius_norm_squared(tt, differentiable=False, + name='frobenius_norm_squared'): """Frobenius norm squared of `TensorTrain` or of each TT in `TensorTrainBatch`. Frobenius norm squared is the sum of squares of all elements in a tensor. @@ -911,6 +931,7 @@ def frobenius_norm_squared(tt, differentiable=False): tt: `TensorTrain` or `TensorTrainBatch` object differentiable: bool, whether to use a differentiable implementation or a fast and stable implementation based on QR decomposition. + name: string, name of the Op. Returns a number which is the Frobenius norm squared of `tt`, if it is `TensorTrain` @@ -918,42 +939,44 @@ def frobenius_norm_squared(tt, differentiable=False): a Tensor of size tt.batch_size, consisting of the Frobenius norms squared of each TensorTrain in `tt`, if it is `TensorTrainBatch` """ - if differentiable: - if hasattr(tt, 'batch_size'): - bs_str = 'n' - else: - bs_str = '' - if tt.is_tt_matrix(): - running_prod = tf.einsum('{0}aijb,{0}cijd->{0}bd'.format(bs_str), - tt.tt_cores[0], tt.tt_cores[0]) - else: - running_prod = tf.einsum('{0}aib,{0}cid->{0}bd'.format(bs_str), - tt.tt_cores[0], tt.tt_cores[0]) - - for core_idx in range(1, tt.ndims()): - curr_core = tt.tt_cores[core_idx] + with tf.name_scope(name, values=tt.tt_cores): + if differentiable: + if hasattr(tt, 'batch_size'): + bs_str = 'n' + else: + bs_str = '' if tt.is_tt_matrix(): - running_prod = tf.einsum('{0}ac,{0}aijb,{0}cijd->{0}bd'.format(bs_str), - running_prod, curr_core, curr_core) + running_prod = tf.einsum('{0}aijb,{0}cijd->{0}bd'.format(bs_str), + tt.tt_cores[0], tt.tt_cores[0]) else: - running_prod = tf.einsum('{0}ac,{0}aib,{0}cid->{0}bd'.format(bs_str), - running_prod, curr_core, curr_core) + running_prod = tf.einsum('{0}aib,{0}cid->{0}bd'.format(bs_str), + tt.tt_cores[0], tt.tt_cores[0]) + + for core_idx in range(1, tt.ndims()): + curr_core = tt.tt_cores[core_idx] + if tt.is_tt_matrix(): + running_prod = tf.einsum('{0}ac,{0}aijb,{0}cijd->{0}bd'.format(bs_str), + running_prod, curr_core, curr_core) + else: + running_prod = tf.einsum('{0}ac,{0}aib,{0}cid->{0}bd'.format(bs_str), + running_prod, curr_core, curr_core) - return tf.squeeze(running_prod, [-1, -2]) + return tf.squeeze(running_prod, [-1, -2]) - else: - orth_tt = decompositions.orthogonalize_tt_cores(tt, left_to_right=True) - # All the cores of orth_tt except the last one are orthogonal, hence - # the Frobenius norm of orth_tt equals to the norm of the last core. - if hasattr(tt, 'batch_size'): - batch_size = shapes.lazy_batch_size(tt) - last_core = tf.reshape(orth_tt.tt_cores[-1], (batch_size, -1)) - return tf.norm(last_core, axis=1) ** 2 else: - return tf.norm(orth_tt.tt_cores[-1]) ** 2 + orth_tt = decompositions.orthogonalize_tt_cores(tt, left_to_right=True) + # All the cores of orth_tt except the last one are orthogonal, hence + # the Frobenius norm of orth_tt equals to the norm of the last core. + if hasattr(tt, 'batch_size'): + batch_size = shapes.lazy_batch_size(tt) + last_core = tf.reshape(orth_tt.tt_cores[-1], (batch_size, -1)) + return tf.norm(last_core, axis=1) ** 2 + else: + return tf.norm(orth_tt.tt_cores[-1]) ** 2 -def frobenius_norm(tt, epsilon=1e-5, differentiable=False): +def frobenius_norm(tt, epsilon=1e-5, differentiable=False, + name='frobenius_norm'): """Frobenius norm of `TensorTrain` or of each TT in `TensorTrainBatch` Frobenius norm is the sqrt of the sum of squares of all elements in a tensor. @@ -964,6 +987,7 @@ def frobenius_norm(tt, epsilon=1e-5, differentiable=False): numerical stability (e.g. gradient of sqrt at zero is inf). differentiable: bool, whether to use a differentiable implementation or a fast and stable implementation based on QR decomposition. + name: string, name of the Op. Returns a number which is the Frobenius norm of `tt`, if it is `TensorTrain` @@ -971,15 +995,17 @@ def frobenius_norm(tt, epsilon=1e-5, differentiable=False): a Tensor of size tt.batch_size, consisting of the Frobenius norms of each TensorTrain in `tt`, if it is `TensorTrainBatch` """ - return tf.sqrt(frobenius_norm_squared(tt, differentiable) + epsilon) + with tf.name_scope(name, values=tt.tt_cores): + return tf.sqrt(frobenius_norm_squared(tt, differentiable) + epsilon) -def transpose(tt_matrix): +def transpose(tt_matrix, name='transpose'): """Transpose a TT-matrix or a batch of TT-matrices. Args: tt_matrix: `TensorTrain` or `TensorTrainBatch` object containing a TT-matrix (or a batch of TT-matrices). + name: string, name of the Op. Returns: `TensorTrain` or `TensorTrainBatch` object containing a transposed TT-matrix @@ -991,27 +1017,28 @@ def transpose(tt_matrix): if not isinstance(tt_matrix, TensorTrainBase) or not tt_matrix.is_tt_matrix(): raise ValueError('The argument should be a TT-matrix.') - transposed_tt_cores = [] - for core_idx in range(tt_matrix.ndims()): - curr_core = tt_matrix.tt_cores[core_idx] + with tf.name_scope(name, values=tt_matrix.tt_cores): + transposed_tt_cores = [] + for core_idx in range(tt_matrix.ndims()): + curr_core = tt_matrix.tt_cores[core_idx] + if isinstance(tt_matrix, TensorTrain): + transposed_tt_cores.append(tf.transpose(curr_core, (0, 2, 1, 3))) + else: + # TensorTrainBatch. + transposed_tt_cores.append(tf.transpose(curr_core, (0, 1, 3, 2, 4))) + + tt_matrix_shape = tt_matrix.get_raw_shape() + transposed_shape = tt_matrix_shape[1], tt_matrix_shape[0] + tt_ranks = tt_matrix.get_tt_ranks() if isinstance(tt_matrix, TensorTrain): - transposed_tt_cores.append(tf.transpose(curr_core, (0, 2, 1, 3))) + return TensorTrain(transposed_tt_cores, transposed_shape, tt_ranks) else: - # TensorTrainBatch. - transposed_tt_cores.append(tf.transpose(curr_core, (0, 1, 3, 2, 4))) - - tt_matrix_shape = tt_matrix.get_raw_shape() - transposed_shape = tt_matrix_shape[1], tt_matrix_shape[0] - tt_ranks = tt_matrix.get_tt_ranks() - if isinstance(tt_matrix, TensorTrain): - return TensorTrain(transposed_tt_cores, transposed_shape, tt_ranks) - else: - batch_size = tt_matrix.batch_size - return TensorTrainBatch(transposed_tt_cores, transposed_shape, tt_ranks, - batch_size) + batch_size = tt_matrix.batch_size + return TensorTrainBatch(transposed_tt_cores, transposed_shape, tt_ranks, + batch_size) -def quadratic_form(A, b, c): +def quadratic_form(A, b, c, name='quadratic_form'): """Quadratic form b^t A c; A is a TT-matrix, b and c can be batches. Args: @@ -1020,6 +1047,7 @@ def quadratic_form(A, b, c): or `TensorTrainBatch` with a batch of TT-matrices of size N x 1. c: `TensorTrain` object containing a TT-matrix of size M x 1 or `TensorTrainBatch` with a batch of TT-matrices of size M x 1. + name: string, name of the Op. Returns: A number, the value of the quadratic form if all the arguments are @@ -1055,62 +1083,65 @@ def quadratic_form(A, b, c): c_bs_str = 'p' if c_is_batch else '' out_bs_str = 'p' if b_is_batch or c_is_batch else '' - ndims = A.ndims() - curr_core_1 = b.tt_cores[0] - curr_core_2 = c.tt_cores[0] - curr_matrix_core = A.tt_cores[0] - # We enumerate the dummy dimension (that takes 1 value) with `k`. - # You may think that using two different k would be faster, but in my - # experience it's even a little bit slower (but neglectable in general). - einsum_str = '{0}aikb,cijd,{1}ejkf->{2}bdf'.format(b_bs_str, c_bs_str, - out_bs_str) - res = tf.einsum(einsum_str, curr_core_1, curr_matrix_core, curr_core_2) - for core_idx in range(1, ndims): - curr_core_1 = b.tt_cores[core_idx] - curr_core_2 = c.tt_cores[core_idx] - curr_matrix_core = A.tt_cores[core_idx] - einsum_str = '{2}ace,{0}aikb,cijd,{1}ejkf->{2}bdf'.format(b_bs_str, - c_bs_str, - out_bs_str) - res = tf.einsum(einsum_str, res, curr_core_1, - curr_matrix_core, curr_core_2) - - # Squeeze to make the result a number instead of 1 x 1 for NON batch case and - # to make the result a tensor of size - # batch_size - # instead of - # batch_size x 1 x 1 - # in the batch case. - return tf.squeeze(res) - - -def cast(tt_a, dtype): + with tf.name_scope(name, values=A.tt_cores+b.tt_cores+c.tt_cores): + ndims = A.ndims() + curr_core_1 = b.tt_cores[0] + curr_core_2 = c.tt_cores[0] + curr_matrix_core = A.tt_cores[0] + # We enumerate the dummy dimension (that takes 1 value) with `k`. + # You may think that using two different k would be faster, but in my + # experience it's even a little bit slower (but neglectable in general). + einsum_str = '{0}aikb,cijd,{1}ejkf->{2}bdf'.format(b_bs_str, c_bs_str, + out_bs_str) + res = tf.einsum(einsum_str, curr_core_1, curr_matrix_core, curr_core_2) + for core_idx in range(1, ndims): + curr_core_1 = b.tt_cores[core_idx] + curr_core_2 = c.tt_cores[core_idx] + curr_matrix_core = A.tt_cores[core_idx] + einsum_str = '{2}ace,{0}aikb,cijd,{1}ejkf->{2}bdf'.format(b_bs_str, + c_bs_str, + out_bs_str) + res = tf.einsum(einsum_str, res, curr_core_1, + curr_matrix_core, curr_core_2) + + # Squeeze to make the result a number instead of 1 x 1 for NON batch case + # and to make the result a tensor of size + # batch_size + # instead of + # batch_size x 1 x 1 + # in the batch case. + return tf.squeeze(res) + + +def cast(tt, dtype, name='cast'): """Casts a tt-tensor to a new type. Args: - tt_a: `TensorTrain` object. + tt: `TensorTrain` object. dtype: The destination type. + name: string, name of the Op. Raises: - TypeError: If `tt_a` cannot be cast to the `dtype`. - ValueError: If `tt_a` is not a `TensorTrain` or `TensorTrainBatch`. + TypeError: If `tt` cannot be cast to the `dtype`. + ValueError: If `tt` is not a `TensorTrain` or `TensorTrainBatch`. """ - res_cores = [] - cores = tt_a.tt_cores - for core_idx in range(tt_a.ndims()): - res_cores.append(tf.cast(cores[core_idx], dtype)) - res_shape = tt_a.get_raw_shape() - res_ranks = tt_a.get_tt_ranks() - if isinstance(tt_a, TensorTrain): - return TensorTrain(res_cores, res_shape, res_ranks) - elif isinstance(tt_a, TensorTrainBatch): - return TensorTrainBatch(res_cores, res_shape, res_ranks, tt_a.batch_size) - else: - raise ValueError('Unsupported type of input "%s", should be TensorTrain or ' - 'TensorTrainBatch.' % tt_a) + with tf.name_scope(name, values=tt.tt_cores): + res_cores = [] + cores = tt.tt_cores + for core_idx in range(tt.ndims()): + res_cores.append(tf.cast(cores[core_idx], dtype)) + res_shape = tt.get_raw_shape() + res_ranks = tt.get_tt_ranks() + if isinstance(tt, TensorTrain): + return TensorTrain(res_cores, res_shape, res_ranks) + elif isinstance(tt, TensorTrainBatch): + return TensorTrainBatch(res_cores, res_shape, res_ranks, tt.batch_size) + else: + raise ValueError('Unsupported type of input "%s", should be TensorTrain ' + 'or TensorTrainBatch.' % tt) -def gather_nd(tt, indices): +def gather_nd(tt, indices, name='gather_nd'): """out[i] = tt[indices[i, 0], indices[i, 1], ...] Equivalent to @@ -1127,6 +1158,7 @@ def gather_nd(tt, indices): dimensions in TT: indices.shape[-1] = tt.ndims for `TensorTrain` indices.shape[-1] = tt.ndims + 1 for `TensorTrainBatch` + name: string, name of the Op. Returns: tf.Tensor with elements specified by indices. @@ -1135,38 +1167,39 @@ def gather_nd(tt, indices): ValueError if `indices` have wrong shape. NotImplementedError if `tt` is a TT-matrix. """ - if tt.is_tt_matrix(): - raise NotImplementedError('gather_nd doesnt support TT-matrices yet ' - '(got %s)' % tt) - indices = tf.convert_to_tensor(indices) - if isinstance(tt, TensorTrainBatch): - if indices.get_shape()[-1] != tt.ndims() + 1: - raise ValueError('The last dimension of indices (%d) should have ' - 'the same size as the number of dimensions in the tt ' - 'object (%d) + 1 (for the batch dimension).' % - (indices.get_shape()[-1], tt.ndims())) - else: - if indices.get_shape()[-1] != tt.ndims(): - raise ValueError('The last dimension of indices (%d) should have ' - 'the same size as the number of dimensions in the tt ' - 'object (%d).' % (indices.get_shape()[-1], tt.ndims())) - tt_elements = tf.ones(tf.shape(indices)[:-1]) - tt_elements = tf.reshape(tt_elements, (-1, 1, 1)) - for core_idx in range(tt.ndims()): - curr_core = tt.tt_cores[core_idx] + with tf.name_scope(name, values=tt.tt_cores+[indices]): + if tt.is_tt_matrix(): + raise NotImplementedError('gather_nd doesnt support TT-matrices yet ' + '(got %s)' % tt) + indices = tf.convert_to_tensor(indices) if isinstance(tt, TensorTrainBatch): - curr_core = tf.transpose(curr_core, (0, 2, 1, 3)) - curr_idx = tf.stack((indices[:, 0], indices[:, core_idx + 1]), axis=1) - core_slices = tf.gather_nd(curr_core, curr_idx) + if indices.get_shape()[-1] != tt.ndims() + 1: + raise ValueError('The last dimension of indices (%d) should have ' + 'the same size as the number of dimensions in the tt ' + 'object (%d) + 1 (for the batch dimension).' % + (indices.get_shape()[-1], tt.ndims())) else: - curr_core = tf.transpose(curr_core, (1, 0, 2)) - core_slices = tf.gather(curr_core, indices[:, core_idx]) - tt_elements = tf.matmul(tt_elements, core_slices) - tt_elements = tf.reshape(tt_elements, tf.shape(indices)[:-1]) - return tt_elements + if indices.get_shape()[-1] != tt.ndims(): + raise ValueError('The last dimension of indices (%d) should have ' + 'the same size as the number of dimensions in the tt ' + 'object (%d).' % (indices.get_shape()[-1], tt.ndims())) + tt_elements = tf.ones(tf.shape(indices)[:-1]) + tt_elements = tf.reshape(tt_elements, (-1, 1, 1)) + for core_idx in range(tt.ndims()): + curr_core = tt.tt_cores[core_idx] + if isinstance(tt, TensorTrainBatch): + curr_core = tf.transpose(curr_core, (0, 2, 1, 3)) + curr_idx = tf.stack((indices[:, 0], indices[:, core_idx + 1]), axis=1) + core_slices = tf.gather_nd(curr_core, curr_idx) + else: + curr_core = tf.transpose(curr_core, (1, 0, 2)) + core_slices = tf.gather(curr_core, indices[:, core_idx]) + tt_elements = tf.matmul(tt_elements, core_slices) + tt_elements = tf.reshape(tt_elements, tf.shape(indices)[:-1]) + return tt_elements -def renormalize_tt_cores(tt, epsilon=1e-8): +def renormalize_tt_cores(tt, epsilon=1e-8, name='renormalize_tt_cores'): """Renormalizes TT-cores to make them of the same Frobenius norm. Doesn't change the tensor represented by `tt` object, but renormalizes the @@ -1175,41 +1208,44 @@ def renormalize_tt_cores(tt, epsilon=1e-8): Args: tt: `TensorTrain` or `TensorTrainBatch` object epsilon: parameter for numerical stability of sqrt + name: string, name of the Op. + Returns: `TensorTrain` or `TensorTrainBatch` which represents the same tensor as tt, but with all cores having equal norm. In the batch case applies to each TT in `TensorTrainBatch`. - """ - if isinstance(tt, TensorTrain): - new_cores = [] - running_log_norm = 0 - core_norms = [] - for core in tt.tt_cores: - cur_core_norm = tf.sqrt(tf.maximum(tf.reduce_sum(core ** 2), epsilon)) - core_norms.append(cur_core_norm) - running_log_norm += tf.log(cur_core_norm) - - running_log_norm = running_log_norm / tt.ndims() - fact = tf.exp(running_log_norm) - for i, core in enumerate(tt.tt_cores): - new_cores.append(core * fact / core_norms[i]) - - return TensorTrain(new_cores) - else: - sz = (tt.batch_size,) + (len(tt.tt_cores[0].shape) - 1) * (1,) - running_core_log_norms = tf.zeros(sz) - ax = np.arange(len(tt.tt_cores[0].shape))[1:] - fact_list = [] - for core in tt.tt_cores: - cur_core_norm_sq = tf.reduce_sum(core**2, axis=ax, keep_dims=True) - cur_core_norm = tf.sqrt(tf.maximum(epsilon, cur_core_norm_sq)) - fact_list.append(cur_core_norm) - running_core_log_norms += tf.log(cur_core_norm) - - new_cores = [] - exp_fact = tf.exp(running_core_log_norms / tt.ndims()) - for i, core in enumerate(tt.tt_cores): - new_cores.append(tf.multiply(core, exp_fact / fact_list[i])) - - return TensorTrainBatch(new_cores) + # TODO: bad way to check if batch or not. + with tf.name_scope(name, values=tt.tt_cores): + if isinstance(tt, TensorTrain): + new_cores = [] + running_log_norm = 0 + core_norms = [] + for core in tt.tt_cores: + cur_core_norm = tf.sqrt(tf.maximum(tf.reduce_sum(core ** 2), epsilon)) + core_norms.append(cur_core_norm) + running_log_norm += tf.log(cur_core_norm) + + running_log_norm = running_log_norm / tt.ndims() + fact = tf.exp(running_log_norm) + for i, core in enumerate(tt.tt_cores): + new_cores.append(core * fact / core_norms[i]) + + return TensorTrain(new_cores) + else: + sz = (tt.batch_size,) + (len(tt.tt_cores[0].shape) - 1) * (1,) + running_core_log_norms = tf.zeros(sz) + ax = np.arange(len(tt.tt_cores[0].shape))[1:] + fact_list = [] + for core in tt.tt_cores: + cur_core_norm_sq = tf.reduce_sum(core**2, axis=ax, keep_dims=True) + cur_core_norm = tf.sqrt(tf.maximum(epsilon, cur_core_norm_sq)) + fact_list.append(cur_core_norm) + running_core_log_norms += tf.log(cur_core_norm) + + new_cores = [] + exp_fact = tf.exp(running_core_log_norms / tt.ndims()) + for i, core in enumerate(tt.tt_cores): + new_cores.append(tf.multiply(core, exp_fact / fact_list[i])) + + return TensorTrainBatch(new_cores) From 23e570dc26355267d588f7f0f422dfcdeea101dc Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sat, 5 May 2018 20:40:01 +0300 Subject: [PATCH 027/233] Better name for full --- t3f/ops.py | 2 +- t3f/tensor_train_batch.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index a3f10e18..0fa0459a 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -11,7 +11,7 @@ # TODO: add complexities to the comments. -def full(tt, name='tt_to_full'): +def full(tt, name='tt_to_dense'): """Converts a TensorTrain into a regular tensor or matrix (tf.Tensor). Args: diff --git a/t3f/tensor_train_batch.py b/t3f/tensor_train_batch.py index 49bc1ee6..b42d2038 100644 --- a/t3f/tensor_train_batch.py +++ b/t3f/tensor_train_batch.py @@ -13,7 +13,7 @@ class TensorTrainBatch(TensorTrainBase): """ def __init__(self, tt_cores, shape=None, tt_ranks=None, batch_size=None, - convert_to_tensors=True): + convert_to_tensors=True, name="TensorTrainBatch"): """Creates a `TensorTrainBatch`. Args: @@ -31,6 +31,7 @@ def __init__(self, tt_cores, shape=None, tt_ranks=None, batch_size=None, equal to 1. If None, tries to infer the ranks from the cores. convert_to_tensors: bool, if True than convert each element of the tt_cores tuple into a tf.Tensor (e.g. to initialize from np.array) + name: The name of ops. Returns: A `TensorTrainBatch`. @@ -41,8 +42,7 @@ def __init__(self, tt_cores, shape=None, tt_ranks=None, batch_size=None, """ tt_cores = list(tt_cores) if convert_to_tensors: - # TODO: what does this namescope do? - with tf.name_scope("TensorTrainBatch", tt_cores): + with tf.name_scope(name, values=tt_cores): for i in range(len(tt_cores)): name = "core%d" % i tt_cores[i] = tf.convert_to_tensor(tt_cores[i], name=name) From 72483c116b2fa8c7333b3187b3d85192d84634e6 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sat, 5 May 2018 20:44:23 +0300 Subject: [PATCH 028/233] Specify that second argument of name scope is values --- t3f/ops.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index 0fa0459a..6319bb95 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -21,7 +21,7 @@ def full(tt, name='tt_to_dense'): Returns: tf.Tensor. """ - with tf.name_scope(name, tt.tt_cores): + with tf.name_scope(name, values=tt.tt_cores): if isinstance(tt, TensorTrainBatch): # Batch of Tensor Trains. return _full_tt_batch(tt) @@ -300,19 +300,19 @@ def matmul(a, b, name='matmul'): """ # TODO: is it safe to check types? What if a class is derived from TT? if isinstance(a, TensorTrainBase) and isinstance(b, TensorTrainBase): - with tf.name_scope(name, a.tt_cores + b.tt_cores): + with tf.name_scope(name, values=a.tt_cores+b.tt_cores): return tt_tt_matmul(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): - with tf.name_scope(name, a.tt_cores + [b]): + with tf.name_scope(name, values=a.tt_cores+[b]): return tt_dense_matmul(a, b) elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): - with tf.name_scope(name, [a] + b.tt_cores): + with tf.name_scope(name, values=[a]+b.tt_cores): return dense_tt_matmul(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): - with tf.name_scope(name, a.tt_cores + [b]): + with tf.name_scope(name, values=a.tt_cores+[b]): return tt_sparse_matmul(a, b) elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): - with tf.name_scope(name, [a] + b.tt_cores): + with tf.name_scope(name, values=[a]+b.tt_cores): return sparse_tt_matmul(a, b) else: raise ValueError('Argument types are not supported in matmul: %s x %s' % @@ -522,19 +522,19 @@ def flat_inner(a, b, name='flat_inner'): """ # TODO: is it safe to check types? What if a class is derived from TT? if isinstance(a, TensorTrainBase) and isinstance(b, TensorTrainBase): - with tf.name_scope(name, a.tt_cores + b.tt_cores): + with tf.name_scope(name, values=a.tt_cores+b.tt_cores): return tt_tt_flat_inner(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): - with tf.name_scope(name, a.tt_cores + [b]): + with tf.name_scope(name, values=a.tt_cores+[b]): return tt_dense_flat_inner(a, b) elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): - with tf.name_scope(name, [a] + b.tt_cores): + with tf.name_scope(name, values=[a]+b.tt_cores): return dense_tt_flat_inner(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): - with tf.name_scope(name, a.tt_cores + [b]): + with tf.name_scope(name, values=a.tt_cores+[b]): return tt_sparse_flat_inner(a, b) elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): - with tf.name_scope(name, [a] + b.tt_cores): + with tf.name_scope(name, values=[a]+b.tt_cores): return sparse_tt_flat_inner(a, b) else: raise ValueError('Argument types are not supported in flat_inner: %s x %s' % @@ -719,7 +719,7 @@ def add(tt_a, tt_b, name='add'): raise ValueError('The batch sizes are different and not 1, broadcasting is ' 'not available.') - with tf.name_scope(name, values=tt_a.tt_cores + tt_b.tt_cores): + with tf.name_scope(name, values=tt_a.tt_cores+tt_b.tt_cores): is_a_batch = isinstance(tt_a, TensorTrainBatch) is_b_batch = isinstance(tt_b, TensorTrainBatch) is_batch_case = is_a_batch or is_b_batch From e1f1df023ca06f709908e51de80c136d38b98792 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sat, 5 May 2018 21:04:13 +0300 Subject: [PATCH 029/233] Fix: do not concat list and tuple --- t3f/ops.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index 6319bb95..1cad7b34 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -303,16 +303,16 @@ def matmul(a, b, name='matmul'): with tf.name_scope(name, values=a.tt_cores+b.tt_cores): return tt_tt_matmul(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): - with tf.name_scope(name, values=a.tt_cores+[b]): + with tf.name_scope(name, values=a.tt_cores+(b,)): return tt_dense_matmul(a, b) elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): - with tf.name_scope(name, values=[a]+b.tt_cores): + with tf.name_scope(name, values=(a,)+b.tt_cores): return dense_tt_matmul(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): - with tf.name_scope(name, values=a.tt_cores+[b]): + with tf.name_scope(name, values=a.tt_cores+(b,)): return tt_sparse_matmul(a, b) elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): - with tf.name_scope(name, values=[a]+b.tt_cores): + with tf.name_scope(name, values=(a,)+b.tt_cores): return sparse_tt_matmul(a, b) else: raise ValueError('Argument types are not supported in matmul: %s x %s' % @@ -525,16 +525,16 @@ def flat_inner(a, b, name='flat_inner'): with tf.name_scope(name, values=a.tt_cores+b.tt_cores): return tt_tt_flat_inner(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.Tensor): - with tf.name_scope(name, values=a.tt_cores+[b]): + with tf.name_scope(name, values=a.tt_cores+(b,)): return tt_dense_flat_inner(a, b) elif isinstance(a, tf.Tensor) and isinstance(b, TensorTrain): - with tf.name_scope(name, values=[a]+b.tt_cores): + with tf.name_scope(name, values=(a,)+b.tt_cores): return dense_tt_flat_inner(a, b) elif isinstance(a, TensorTrain) and isinstance(b, tf.SparseTensor): - with tf.name_scope(name, values=a.tt_cores+[b]): + with tf.name_scope(name, values=a.tt_cores+(b,)): return tt_sparse_flat_inner(a, b) elif isinstance(a, tf.SparseTensor) and isinstance(b, TensorTrain): - with tf.name_scope(name, values=[a]+b.tt_cores): + with tf.name_scope(name, values=(a,)+b.tt_cores): return sparse_tt_flat_inner(a, b) else: raise ValueError('Argument types are not supported in flat_inner: %s x %s' % @@ -779,7 +779,7 @@ def multiply(tt_left, right, name='multiply'): is_batch_case = is_left_batch or is_right_batch ndims = tt_left.ndims() if not isinstance(right, TensorTrainBase): - with tf.name_scope(name, values=tt_left.tt_cores+[right]): + with tf.name_scope(name, values=tt_left.tt_cores+(right,)): # Assume right is a number, not TensorTrain. # To squash right uniformly across TT-cores we pull its absolute value # and raise to the power 1/ndims. First TT-core is multiplied by the sign @@ -1167,7 +1167,7 @@ def gather_nd(tt, indices, name='gather_nd'): ValueError if `indices` have wrong shape. NotImplementedError if `tt` is a TT-matrix. """ - with tf.name_scope(name, values=tt.tt_cores+[indices]): + with tf.name_scope(name, values=tt.tt_cores+(indices,)): if tt.is_tt_matrix(): raise NotImplementedError('gather_nd doesnt support TT-matrices yet ' '(got %s)' % tt) From 1789947c45d16553a047305e7e79cdeee32e1138 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sat, 5 May 2018 21:07:51 +0300 Subject: [PATCH 030/233] Fix broken multiply by number --- t3f/ops.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index 1cad7b34..7830bf53 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -915,11 +915,11 @@ def multiply(tt_left, right, name='multiply'): combined_ranks = zip(tt_left.get_tt_ranks(), right.get_tt_ranks()) out_ranks = [a * b for a, b in combined_ranks] - if not is_batch_case: - return TensorTrain(tt_cores, tt_left.get_raw_shape(), out_ranks) - else: - return TensorTrainBatch(tt_cores, tt_left.get_raw_shape(), out_ranks, - batch_size=out_batch_size) + if not is_batch_case: + return TensorTrain(tt_cores, tt_left.get_raw_shape(), out_ranks) + else: + return TensorTrainBatch(tt_cores, tt_left.get_raw_shape(), out_ranks, + batch_size=out_batch_size) def frobenius_norm_squared(tt, differentiable=False, name='frobenius_norm_squared'): From 41f5c6ac6585aade1f5393f0cd1206d71d12ce9e Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 6 May 2018 12:11:16 +0300 Subject: [PATCH 031/233] Add t3f_ prefix to op names --- t3f/ops.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index 7830bf53..7af9bf11 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -11,7 +11,7 @@ # TODO: add complexities to the comments. -def full(tt, name='tt_to_dense'): +def full(tt, name='t3f_full'): """Converts a TensorTrain into a regular tensor or matrix (tf.Tensor). Args: @@ -275,7 +275,7 @@ def tt_sparse_matmul(tt_matrix_a, sparse_matrix_b): raise NotImplementedError -def matmul(a, b, name='matmul'): +def matmul(a, b, name='t3f_matmul'): """Multiplies two matrices that can be TT-, dense, or sparse. Note that multiplication of two TT-matrices returns a TT-matrix with much @@ -503,7 +503,7 @@ def sparse_tt_flat_inner(sparse_a, tt_b): raise NotImplementedError -def flat_inner(a, b, name='flat_inner'): +def flat_inner(a, b, name='t3f_flat_inner'): """Inner product along all axis. The shapes of a and b should coincide. @@ -685,7 +685,7 @@ def _add_batch_matrix_cores(tt_a, tt_b): return tt_cores, batch_size -def add(tt_a, tt_b, name='add'): +def add(tt_a, tt_b, name='t3f_add'): """Returns a TensorTrain corresponding to elementwise sum tt_a + tt_b. The shapes of tt_a and tt_b should coincide. @@ -748,7 +748,7 @@ def add(tt_a, tt_b, name='add'): return TensorTrain(tt_cores, tt_a.get_raw_shape(), out_ranks) -def multiply(tt_left, right, name='multiply'): +def multiply(tt_left, right, name='t3f_multiply'): """Returns a TensorTrain corresponding to element-wise product tt_left * right. Supports broadcasting: @@ -922,7 +922,7 @@ def multiply(tt_left, right, name='multiply'): batch_size=out_batch_size) def frobenius_norm_squared(tt, differentiable=False, - name='frobenius_norm_squared'): + name='t3f_frobenius_norm_squared'): """Frobenius norm squared of `TensorTrain` or of each TT in `TensorTrainBatch`. Frobenius norm squared is the sum of squares of all elements in a tensor. @@ -976,7 +976,7 @@ def frobenius_norm_squared(tt, differentiable=False, def frobenius_norm(tt, epsilon=1e-5, differentiable=False, - name='frobenius_norm'): + name='t3f_frobenius_norm'): """Frobenius norm of `TensorTrain` or of each TT in `TensorTrainBatch` Frobenius norm is the sqrt of the sum of squares of all elements in a tensor. @@ -999,7 +999,7 @@ def frobenius_norm(tt, epsilon=1e-5, differentiable=False, return tf.sqrt(frobenius_norm_squared(tt, differentiable) + epsilon) -def transpose(tt_matrix, name='transpose'): +def transpose(tt_matrix, name='t3f_transpose'): """Transpose a TT-matrix or a batch of TT-matrices. Args: @@ -1038,7 +1038,7 @@ def transpose(tt_matrix, name='transpose'): batch_size) -def quadratic_form(A, b, c, name='quadratic_form'): +def quadratic_form(A, b, c, name='t3f_quadratic_form'): """Quadratic form b^t A c; A is a TT-matrix, b and c can be batches. Args: @@ -1113,7 +1113,7 @@ def quadratic_form(A, b, c, name='quadratic_form'): return tf.squeeze(res) -def cast(tt, dtype, name='cast'): +def cast(tt, dtype, name='t3f_cast'): """Casts a tt-tensor to a new type. Args: @@ -1141,7 +1141,7 @@ def cast(tt, dtype, name='cast'): 'or TensorTrainBatch.' % tt) -def gather_nd(tt, indices, name='gather_nd'): +def gather_nd(tt, indices, name='t3f_gather_nd'): """out[i] = tt[indices[i, 0], indices[i, 1], ...] Equivalent to @@ -1199,7 +1199,7 @@ def gather_nd(tt, indices, name='gather_nd'): return tt_elements -def renormalize_tt_cores(tt, epsilon=1e-8, name='renormalize_tt_cores'): +def renormalize_tt_cores(tt, epsilon=1e-8, name='t3f_renormalize_tt_cores'): """Renormalizes TT-cores to make them of the same Frobenius norm. Doesn't change the tensor represented by `tt` object, but renormalizes the From 4397177750600064505effed2a6cf2ca642d9064 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 6 May 2018 12:43:03 +0300 Subject: [PATCH 032/233] Add name scopes to batch ops --- t3f/batch_ops.py | 211 +++++++++++++++++++++++++---------------------- 1 file changed, 114 insertions(+), 97 deletions(-) diff --git a/t3f/batch_ops.py b/t3f/batch_ops.py index bd89bb36..abae15a8 100644 --- a/t3f/batch_ops.py +++ b/t3f/batch_ops.py @@ -1,3 +1,4 @@ +import itertools import tensorflow as tf from t3f.tensor_train_base import TensorTrainBase @@ -5,11 +6,12 @@ from t3f import ops -def concat_along_batch_dim(tt_list): +def concat_along_batch_dim(tt_list, name='t3f_concat_along_batch_dim'): """Concat all TensorTrainBatch objects along batch dimension. Args: tt_list: a list of TensorTrainBatch objects. + name: string, name of the Op. Returns: TensorTrainBatch @@ -34,46 +36,52 @@ def concat_along_batch_dim(tt_list): '%s and %s' % (tt_list[0].get_tt_ranks(), tt_list[batch_idx].get_tt_ranks())) - res_cores = [] - for core_idx in range(ndims): - curr_core = tf.concat([tt.tt_cores[core_idx] for tt in tt_list], axis=0) - res_cores.append(curr_core) + list_of_cores_lists = [tt.tt_cores for tt in tt_list] + all_cores = tuple(itertools.chain.from_iterable(list_of_cores_lists)) + with tf.name_scope(name, values=all_cores): + res_cores = [] + for core_idx in range(ndims): + curr_core = tf.concat([tt.tt_cores[core_idx] for tt in tt_list], axis=0) + res_cores.append(curr_core) - try: - batch_size = sum([tt.batch_size for tt in tt_list]) - except TypeError: - # The batch sizes are not defined and you can't sum Nones. - batch_size = None + try: + batch_size = sum([tt.batch_size for tt in tt_list]) + except TypeError: + # The batch sizes are not defined and you can't sum Nones. + batch_size = None - return TensorTrainBatch(res_cores, tt_list[0].get_raw_shape(), - tt_list[0].get_tt_ranks(), batch_size) + return TensorTrainBatch(res_cores, tt_list[0].get_raw_shape(), + tt_list[0].get_tt_ranks(), batch_size) -def multiply_along_batch_dim(batch_tt, weights): +def multiply_along_batch_dim(batch_tt, weights, + name='t3f_multiply_along_batch_dim'): """Multiply each TensorTrain in a batch by a number. Args: batch_tt: TensorTrainBatch object, TT-matrices or TT-tensors. weights: 1-D tf.Tensor (or something convertible to it like np.array) of size tt.batch_size with weights. + name: string, name of the Op. Returns: TensorTrainBatch """ - weights = tf.convert_to_tensor(weights) - tt_cores = list(batch_tt.tt_cores) - if batch_tt.is_tt_matrix(): - weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis, tf.newaxis] - else: - weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis] - tt_cores[0] = weights * tt_cores[0] - out_shape = batch_tt.get_raw_shape() - out_ranks = batch_tt.get_tt_ranks() - out_batch_size = batch_tt.batch_size - return TensorTrainBatch(tt_cores, out_shape, out_ranks, out_batch_size) - - -def gram_matrix(tt_vectors, matrix=None): + with tf.name_scope(name, values=batch_tt.tt_cores+(weights,)): + weights = tf.convert_to_tensor(weights) + tt_cores = list(batch_tt.tt_cores) + if batch_tt.is_tt_matrix(): + weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis, tf.newaxis] + else: + weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis] + tt_cores[0] = weights * tt_cores[0] + out_shape = batch_tt.get_raw_shape() + out_ranks = batch_tt.get_tt_ranks() + out_batch_size = batch_tt.batch_size + return TensorTrainBatch(tt_cores, out_shape, out_ranks, out_batch_size) + + +def gram_matrix(tt_vectors, matrix=None, name='t3f_gram_matrix'): """Computes Gramian matrix of a batch of TT-vectors. If matrix is None, computes @@ -87,6 +95,7 @@ def gram_matrix(tt_vectors, matrix=None): Args: tt_vectors: TensorTrainBatch. matrix: None, or TensorTrain matrix. + name: string, name of the Op. Returns: tf.tensor with the Gram matrix. @@ -108,7 +117,8 @@ def gram_matrix(tt_vectors, matrix=None): return pairwise_flat_inner(tt_vectors, tt_vectors, matrix) -def pairwise_flat_inner(tt_1, tt_2, matrix=None): +def pairwise_flat_inner(tt_1, tt_2, matrix=None, + name='t3f_pairwise_flat_inner'): """Computes all scalar products between two batches of TT-objects. If matrix is None, computes @@ -124,6 +134,7 @@ def pairwise_flat_inner(tt_1, tt_2, matrix=None): tt_1: TensorTrainBatch. tt_2: TensorTrainBatch. matrix: None, or TensorTrain matrix. + name: string, name of the Op. Returns: tf.tensor with the matrix of pairwise scalar products (flat inners). @@ -147,72 +158,78 @@ def pairwise_flat_inner(tt_1, tt_2, matrix=None): tt_1 is of shape (n, n, ..., n) and is of the TT-rank r1; tt_2 is of shape (m, m, ..., m) and is of the TT-rank r2; """ - ndims = tt_1.ndims() - if matrix is None: - curr_core_1 = tt_1.tt_cores[0] - curr_core_2 = tt_2.tt_cores[0] - mode_string = 'ij' if tt_1.is_tt_matrix() else 'i' - einsum_str = 'pa{0}b,qc{0}d->pqbd'.format(mode_string) - res = tf.einsum(einsum_str, curr_core_1, curr_core_2) - for core_idx in range(1, ndims): - curr_core_1 = tt_1.tt_cores[core_idx] - curr_core_2 = tt_2.tt_cores[core_idx] - einsum_str = 'pqac,pa{0}b,qc{0}d->pqbd'.format(mode_string) - res = tf.einsum(einsum_str, res, curr_core_1, curr_core_2) - else: - # res[i, j] = tt_1[i] ^ T * matrix * tt_2[j] - if not tt_1.is_tt_matrix() or not tt_2.is_tt_matrix() or not matrix.is_tt_matrix(): - raise ValueError('When passing three arguments to pairwise_flat_inner, ' - 'the first 2 of them should be TT-vecors and the last ' - 'should be a TT-matrix. Got %s, %s, and %s instead.' % - (tt_1, tt_2, matrix)) - matrix_shape = matrix.get_raw_shape() - if not tt_1.get_raw_shape()[0].is_compatible_with(matrix_shape[0]): - raise ValueError('The shape of the first argument should be compatible ' - 'with the shape of the TT-matrix, that is it should be ' - 'possible to do the following matmul: ' - 'transpose(tt_1) * matrix. Got the first argument ' - '"%s" and matrix "%s"' % (tt_1, matrix)) - if not tt_2.get_raw_shape()[0].is_compatible_with(matrix_shape[1]): - raise ValueError('The shape of the second argument should be compatible ' - 'with the shape of the TT-matrix, that is it should be ' - 'possible to do the following matmul: ' - 'matrix * tt_2. Got the second argument ' - '"%s" and matrix "%s"' % (tt_2, matrix)) - - vectors_1_shape = tt_1.get_shape() - if vectors_1_shape[2] == 1 and vectors_1_shape[1] != 1: - # TODO: not very efficient, better to use different order in einsum. - tt_1 = ops.transpose(tt_1) - vectors_1_shape = tt_1.get_shape() - vectors_2_shape = tt_2.get_shape() - if vectors_2_shape[2] == 1 and vectors_2_shape[1] != 1: - # TODO: not very efficient, better to use different order in einsum. - tt_2 = ops.transpose(tt_2) - vectors_2_shape = tt_2.get_shape() - if vectors_1_shape[1] != 1: - # TODO: do something so that in case the shape is undefined on compilation - # it still works. - raise ValueError('The tt_vectors_1 argument should be vectors (not ' - 'matrices) with shape defined on compilation.') - if vectors_2_shape[1] != 1: - # TODO: do something so that in case the shape is undefined on compilation - # it still works. - raise ValueError('The tt_vectors_2 argument should be vectors (not ' - 'matrices) with shape defined on compilation.') - curr_core_1 = tt_1.tt_cores[0] - curr_core_2 = tt_2.tt_cores[0] - curr_matrix_core = matrix.tt_cores[0] - # We enumerate the dummy dimension (that takes 1 value) with `k`. - res = tf.einsum('pakib,cijd,qekjf->pqbdf', curr_core_1, curr_matrix_core, - curr_core_2) - for core_idx in range(1, ndims): - curr_core_1 = tt_1.tt_cores[core_idx] - curr_core_2 = tt_2.tt_cores[core_idx] - curr_matrix_core = matrix.tt_cores[core_idx] - res = tf.einsum('pqace,pakib,cijd,qekjf->pqbdf', res, curr_core_1, - curr_matrix_core, curr_core_2) - - # Squeeze to make the result of size batch_size x batch_size instead of - # batch_size x batch_size x 1 x 1. - return tf.squeeze(res) + all_cores = tt_1.tt_cores + tt_2.tt_cores + if matrix is not None: + all_cores += matrix.tt_cores + with tf.name_scope(name, values=all_cores): + ndims = tt_1.ndims() + if matrix is None: + curr_core_1 = tt_1.tt_cores[0] + curr_core_2 = tt_2.tt_cores[0] + mode_string = 'ij' if tt_1.is_tt_matrix() else 'i' + einsum_str = 'pa{0}b,qc{0}d->pqbd'.format(mode_string) + res = tf.einsum(einsum_str, curr_core_1, curr_core_2) + for core_idx in range(1, ndims): + curr_core_1 = tt_1.tt_cores[core_idx] + curr_core_2 = tt_2.tt_cores[core_idx] + einsum_str = 'pqac,pa{0}b,qc{0}d->pqbd'.format(mode_string) + res = tf.einsum(einsum_str, res, curr_core_1, curr_core_2) + else: + # res[i, j] = tt_1[i] ^ T * matrix * tt_2[j] + are_all_maatrices = tt_1.is_tt_matrix() and tt_2.is_tt_matrix() + are_all_maatrices = are_all_maatrices and matrix.is_tt_matrix() + if not are_all_maatrices: + raise ValueError('When passing three arguments to pairwise_flat_inner, ' + 'the first 2 of them should be TT-vecors and the last ' + 'should be a TT-matrix. Got %s, %s, and %s instead.' % + (tt_1, tt_2, matrix)) + matrix_shape = matrix.get_raw_shape() + if not tt_1.get_raw_shape()[0].is_compatible_with(matrix_shape[0]): + raise ValueError('The shape of the first argument should be compatible ' + 'with the shape of the TT-matrix, that is it should ' + 'be possible to do the following matmul: ' + 'transpose(tt_1) * matrix. Got the first argument ' + '"%s" and matrix "%s"' % (tt_1, matrix)) + if not tt_2.get_raw_shape()[0].is_compatible_with(matrix_shape[1]): + raise ValueError('The shape of the second argument should be ' + 'compatible with the shape of the TT-matrix, that is ' + 'it should be possible to do the following matmul: ' + 'matrix * tt_2. Got the second argument ' + '"%s" and matrix "%s"' % (tt_2, matrix)) + + vectors_1_shape = tt_1.get_shape() + if vectors_1_shape[2] == 1 and vectors_1_shape[1] != 1: + # TODO: not very efficient, better to use different order in einsum. + tt_1 = ops.transpose(tt_1) + vectors_1_shape = tt_1.get_shape() + vectors_2_shape = tt_2.get_shape() + if vectors_2_shape[2] == 1 and vectors_2_shape[1] != 1: + # TODO: not very efficient, better to use different order in einsum. + tt_2 = ops.transpose(tt_2) + vectors_2_shape = tt_2.get_shape() + if vectors_1_shape[1] != 1: + # TODO: do something so that in case the shape is undefined on compilation + # it still works. + raise ValueError('The tt_vectors_1 argument should be vectors (not ' + 'matrices) with shape defined on compilation.') + if vectors_2_shape[1] != 1: + # TODO: do something so that in case the shape is undefined on compilation + # it still works. + raise ValueError('The tt_vectors_2 argument should be vectors (not ' + 'matrices) with shape defined on compilation.') + curr_core_1 = tt_1.tt_cores[0] + curr_core_2 = tt_2.tt_cores[0] + curr_matrix_core = matrix.tt_cores[0] + # We enumerate the dummy dimension (that takes 1 value) with `k`. + res = tf.einsum('pakib,cijd,qekjf->pqbdf', curr_core_1, curr_matrix_core, + curr_core_2) + for core_idx in range(1, ndims): + curr_core_1 = tt_1.tt_cores[core_idx] + curr_core_2 = tt_2.tt_cores[core_idx] + curr_matrix_core = matrix.tt_cores[core_idx] + res = tf.einsum('pqace,pakib,cijd,qekjf->pqbdf', res, curr_core_1, + curr_matrix_core, curr_core_2) + + # Squeeze to make the result of size batch_size x batch_size instead of + # batch_size x batch_size x 1 x 1. + return tf.squeeze(res) From 22019ca7ec0ebb1e60ac7f1ea88c34267684a4f0 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 6 May 2018 12:54:57 +0300 Subject: [PATCH 033/233] Add name scopes to decompositions --- t3f/decompositions.py | 217 ++++++++++++++++++++++-------------------- 1 file changed, 114 insertions(+), 103 deletions(-) diff --git a/t3f/decompositions.py b/t3f/decompositions.py index b16ee7f0..3ce28c8c 100644 --- a/t3f/decompositions.py +++ b/t3f/decompositions.py @@ -6,7 +6,8 @@ from t3f import shapes -def to_tt_matrix(mat, shape, max_tt_rank=10, epsilon=None): +def to_tt_matrix(mat, shape, max_tt_rank=10, epsilon=None, + name='t3f_to_tt_matrix'): """Converts a given matrix or vector to a TT-matrix. The matrix dimensions should factorize into d numbers. @@ -46,6 +47,7 @@ def to_tt_matrix(mat, shape, max_tt_rank=10, epsilon=None): the TT-ranks of the result undefined on the compilation stage (e.g. res.get_tt_ranks() will return None, but t3f.tt_ranks(res).eval() will work). + name: string, name of the Op. Returns: `TensorTrain` object containing a TT-matrix. @@ -55,46 +57,48 @@ def to_tt_matrix(mat, shape, max_tt_rank=10, epsilon=None): not a vector of length d + 1 where d is the number of dimensions (rank) of the input tensor, if epsilon is less than 0. """ - mat = tf.convert_to_tensor(mat) - # In case the shape is immutable. - shape = list(shape) - # In case shape represents a vector, e.g. [None, [2, 2, 2]] - if shape[0] is None: - shape[0] = np.ones(len(shape[1])).astype(int) - # In case shape represents a vector, e.g. [[2, 2, 2], None] - if shape[1] is None: - shape[1] = np.ones(len(shape[0])).astype(int) - - shape = np.array(shape) - tens = tf.reshape(mat, shape.flatten()) - d = len(shape[0]) - # transpose_idx = 0, d, 1, d+1 ... - transpose_idx = np.arange(2 * d).reshape(2, d).T.flatten() - transpose_idx = transpose_idx.astype(int) - tens = tf.transpose(tens, transpose_idx) - new_shape = np.prod(shape, axis=0) - tens = tf.reshape(tens, new_shape) - tt_tens = to_tt_tensor(tens, max_tt_rank, epsilon) - tt_cores = [] - static_tt_ranks = tt_tens.get_tt_ranks() - dynamic_tt_ranks = shapes.tt_ranks(tt_tens) - for core_idx in range(d): - curr_core = tt_tens.tt_cores[core_idx] - curr_rank = static_tt_ranks[core_idx].value - if curr_rank is None: - curr_rank = dynamic_tt_ranks[core_idx] - next_rank = static_tt_ranks[core_idx + 1].value - if next_rank is None: - next_rank = dynamic_tt_ranks[core_idx + 1] - curr_core_new_shape = (curr_rank, shape[0, core_idx], - shape[1, core_idx], next_rank) - curr_core = tf.reshape(curr_core, curr_core_new_shape) - tt_cores.append(curr_core) - return TensorTrain(tt_cores, shape, tt_tens.get_tt_ranks()) + with tf.name_scope(name, values=(mat,)): + mat = tf.convert_to_tensor(mat) + # In case the shape is immutable. + shape = list(shape) + # In case shape represents a vector, e.g. [None, [2, 2, 2]] + if shape[0] is None: + shape[0] = np.ones(len(shape[1])).astype(int) + # In case shape represents a vector, e.g. [[2, 2, 2], None] + if shape[1] is None: + shape[1] = np.ones(len(shape[0])).astype(int) + + shape = np.array(shape) + tens = tf.reshape(mat, shape.flatten()) + d = len(shape[0]) + # transpose_idx = 0, d, 1, d+1 ... + transpose_idx = np.arange(2 * d).reshape(2, d).T.flatten() + transpose_idx = transpose_idx.astype(int) + tens = tf.transpose(tens, transpose_idx) + new_shape = np.prod(shape, axis=0) + tens = tf.reshape(tens, new_shape) + tt_tens = to_tt_tensor(tens, max_tt_rank, epsilon) + tt_cores = [] + static_tt_ranks = tt_tens.get_tt_ranks() + dynamic_tt_ranks = shapes.tt_ranks(tt_tens) + for core_idx in range(d): + curr_core = tt_tens.tt_cores[core_idx] + curr_rank = static_tt_ranks[core_idx].value + if curr_rank is None: + curr_rank = dynamic_tt_ranks[core_idx] + next_rank = static_tt_ranks[core_idx + 1].value + if next_rank is None: + next_rank = dynamic_tt_ranks[core_idx + 1] + curr_core_new_shape = (curr_rank, shape[0, core_idx], + shape[1, core_idx], next_rank) + curr_core = tf.reshape(curr_core, curr_core_new_shape) + tt_cores.append(curr_core) + return TensorTrain(tt_cores, shape, tt_tens.get_tt_ranks()) # TODO: implement epsilon. -def to_tt_tensor(tens, max_tt_rank=10, epsilon=None): +def to_tt_tensor(tens, max_tt_rank=10, epsilon=None, + name='t3f_to_tt_tensor'): """Converts a given tf.Tensor to a TT-tensor of the same shape. Args: @@ -122,6 +126,7 @@ def to_tt_tensor(tens, max_tt_rank=10, epsilon=None): the TT-ranks of the result undefined on the compilation stage (e.g. res.get_tt_ranks() will return None, but t3f.tt_ranks(res).eval() will work). + name: string, name of the Op. Returns: `TensorTrain` object containing a TT-tensor. @@ -132,63 +137,65 @@ def to_tt_tensor(tens, max_tt_rank=10, epsilon=None): and not a vector of length d + 1 where d is the number of dimensions (rank) of the input tensor, if epsilon is less than 0. """ - tens = tf.convert_to_tensor(tens) - static_shape = tens.get_shape() - dynamic_shape = tf.shape(tens) - # Raises ValueError if ndims is not defined. - d = static_shape.__len__() - max_tt_rank = np.array(max_tt_rank).astype(np.int32) - if max_tt_rank < 1: - raise ValueError('Maximum TT-rank should be greater or equal to 1.') - if epsilon is not None and epsilon < 0: - raise ValueError('Epsilon should be non-negative.') - if max_tt_rank.size == 1: - max_tt_rank = (max_tt_rank * np.ones(d+1)).astype(np.int32) - elif max_tt_rank.size != d + 1: - raise ValueError('max_tt_rank should be a number or a vector of size (d+1) ' - 'where d is the number of dimensions (rank) of the tensor.') - ranks = [1] * (d + 1) - tt_cores = [] - are_tt_ranks_defined = True - for core_idx in range(d - 1): - curr_mode = static_shape[core_idx].value - if curr_mode is None: - curr_mode = dynamic_shape[core_idx] - rows = ranks[core_idx] * curr_mode - tens = tf.reshape(tens, [rows, -1]) - columns = tens.get_shape()[1].value - if columns is None: - columns = tf.shape(tens)[1] - s, u, v = tf.svd(tens, full_matrices=False) - if max_tt_rank[core_idx + 1] == 1: - ranks[core_idx + 1] = 1 - else: - try: - ranks[core_idx + 1] = min(max_tt_rank[core_idx + 1], rows, columns) - except TypeError: - # Some of the values are undefined on the compilation stage and thus - # they are tf.tensors instead of values. - min_dim = tf.minimum(rows, columns) - ranks[core_idx + 1] = tf.minimum(max_tt_rank[core_idx + 1], min_dim) - are_tt_ranks_defined = False - u = u[:, 0:ranks[core_idx + 1]] - s = s[0:ranks[core_idx + 1]] - v = v[:, 0:ranks[core_idx + 1]] - core_shape = (ranks[core_idx], curr_mode, ranks[core_idx + 1]) - tt_cores.append(tf.reshape(u, core_shape)) - tens = tf.matmul(tf.diag(s), tf.transpose(v)) - last_mode = static_shape[-1].value - if last_mode is None: - last_mode = dynamic_shape[-1] - core_shape = (ranks[d - 1], last_mode, ranks[d]) - tt_cores.append(tf.reshape(tens, core_shape)) - if not are_tt_ranks_defined: - ranks = None - return TensorTrain(tt_cores, static_shape, ranks) + with tf.name_scope(name, values=(tens,)): + tens = tf.convert_to_tensor(tens) + static_shape = tens.get_shape() + dynamic_shape = tf.shape(tens) + # Raises ValueError if ndims is not defined. + d = static_shape.__len__() + max_tt_rank = np.array(max_tt_rank).astype(np.int32) + if max_tt_rank < 1: + raise ValueError('Maximum TT-rank should be greater or equal to 1.') + if epsilon is not None and epsilon < 0: + raise ValueError('Epsilon should be non-negative.') + if max_tt_rank.size == 1: + max_tt_rank = (max_tt_rank * np.ones(d+1)).astype(np.int32) + elif max_tt_rank.size != d + 1: + raise ValueError('max_tt_rank should be a number or a vector of size ' + '(d+1) where d is the number of dimensions (rank) of ' + 'the tensor.') + ranks = [1] * (d + 1) + tt_cores = [] + are_tt_ranks_defined = True + for core_idx in range(d - 1): + curr_mode = static_shape[core_idx].value + if curr_mode is None: + curr_mode = dynamic_shape[core_idx] + rows = ranks[core_idx] * curr_mode + tens = tf.reshape(tens, [rows, -1]) + columns = tens.get_shape()[1].value + if columns is None: + columns = tf.shape(tens)[1] + s, u, v = tf.svd(tens, full_matrices=False) + if max_tt_rank[core_idx + 1] == 1: + ranks[core_idx + 1] = 1 + else: + try: + ranks[core_idx + 1] = min(max_tt_rank[core_idx + 1], rows, columns) + except TypeError: + # Some of the values are undefined on the compilation stage and thus + # they are tf.tensors instead of values. + min_dim = tf.minimum(rows, columns) + ranks[core_idx + 1] = tf.minimum(max_tt_rank[core_idx + 1], min_dim) + are_tt_ranks_defined = False + u = u[:, 0:ranks[core_idx + 1]] + s = s[0:ranks[core_idx + 1]] + v = v[:, 0:ranks[core_idx + 1]] + core_shape = (ranks[core_idx], curr_mode, ranks[core_idx + 1]) + tt_cores.append(tf.reshape(u, core_shape)) + tens = tf.matmul(tf.diag(s), tf.transpose(v)) + last_mode = static_shape[-1].value + if last_mode is None: + last_mode = dynamic_shape[-1] + core_shape = (ranks[d - 1], last_mode, ranks[d]) + tt_cores.append(tf.reshape(tens, core_shape)) + if not are_tt_ranks_defined: + ranks = None + return TensorTrain(tt_cores, static_shape, ranks) # TODO: rename round so not to shadow python.round? -def round(tt, max_tt_rank=None, epsilon=None): +def round(tt, max_tt_rank=None, epsilon=None, name='t3f_round'): """TT-rounding procedure, returns a TT object with smaller TT-ranks. Args: @@ -216,6 +223,7 @@ def round(tt, max_tt_rank=None, epsilon=None): the TT-ranks of the result undefined on the compilation stage (e.g. res.get_tt_ranks() will return None, but t3f.tt_ranks(res).eval() will work). + name: string, name of the Op. Returns: `TensorTrain` object containing a TT-tensor. @@ -382,27 +390,30 @@ def _round_batch_tt(tt, max_tt_rank, epsilon): return TensorTrainBatch(tt_cores, tt.get_raw_shape(), ranks, batch_size=tt.batch_size) -def orthogonalize_tt_cores(tt, left_to_right=True): +def orthogonalize_tt_cores(tt, left_to_right=True, + name='t3f_orthogonalize_tt_cores'): """Orthogonalize TT-cores of a TT-object. Args: tt: TenosorTrain or a TensorTrainBatch. left_to_right: bool, the direction of orthogonalization. + name: string, name of the Op. Returns: The same type as the input `tt` (TenosorTrain or a TensorTrainBatch). """ - if isinstance(tt, TensorTrainBatch): - if left_to_right: - return _orthogonalize_batch_tt_cores_left_to_right(tt) - else: - raise NotImplementedError('Batch right to left orthogonalization is not ' - 'supported yet.') - else: - if left_to_right: - return _orthogonalize_tt_cores_left_to_right(tt) + with tf.name_scope(name, values=tt.tt_cores): + if isinstance(tt, TensorTrainBatch): + if left_to_right: + return _orthogonalize_batch_tt_cores_left_to_right(tt) + else: + raise NotImplementedError('Batch right to left orthogonalization is ' + 'not supported yet.') else: - return _orthogonalize_tt_cores_right_to_left(tt) + if left_to_right: + return _orthogonalize_tt_cores_left_to_right(tt) + else: + return _orthogonalize_tt_cores_right_to_left(tt) def _orthogonalize_tt_cores_left_to_right(tt): From 38905385dccc17d32fa8703062ce4d50fc34dd7b Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 6 May 2018 13:12:07 +0300 Subject: [PATCH 034/233] Add name scopes to approximate and initializers --- t3f/approximate.py | 221 +++++++++++++++++++++++--------------------- t3f/initializers.py | 212 ++++++++++++++++++++++++------------------ 2 files changed, 240 insertions(+), 193 deletions(-) diff --git a/t3f/approximate.py b/t3f/approximate.py index 4cb8f495..96191630 100644 --- a/t3f/approximate.py +++ b/t3f/approximate.py @@ -1,3 +1,4 @@ +import itertools import numpy as np import tensorflow as tf from t3f.tensor_train_batch import TensorTrainBatch @@ -5,7 +6,7 @@ from t3f import batch_ops -def add_n(tt_objects, max_tt_rank): +def add_n(tt_objects, max_tt_rank, name='t3f_approximate_add_n'): """Adds a bunch of TT-object and round after each summation. This version implements a slow-to-compile but fast-to-execute (at least on @@ -17,6 +18,7 @@ def add_n(tt_objects, max_tt_rank): Args: tt_objects: a list of `TensorTrainBase` objects. max_tt_rank: a number, TT-rank for each individual rounding. + name: string, name of the Op. Returns: Object of the same type as each input. @@ -24,19 +26,23 @@ def add_n(tt_objects, max_tt_rank): See Also: t3f.approximate.reduce_sum_batch """ - prev_level = tt_objects - while len(prev_level) > 1: - next_level = [] - for i in range(0, len(prev_level), 2): - curr = prev_level[i] - if i + 1 < len(prev_level): - curr = decompositions.round(curr + prev_level[i + 1], max_tt_rank) - next_level.append(curr) - prev_level = next_level - return prev_level[0] - - -def reduce_sum_batch(tt_batch, max_tt_rank, coef=None): + list_of_cores_lists = [tt.tt_cores for tt in tt_objects] + all_cores = tuple(itertools.chain.from_iterable(list_of_cores_lists)) + with tf.name_scope(name, values=all_cores): + prev_level = tt_objects + while len(prev_level) > 1: + next_level = [] + for i in range(0, len(prev_level), 2): + curr = prev_level[i] + if i + 1 < len(prev_level): + curr = decompositions.round(curr + prev_level[i + 1], max_tt_rank) + next_level.append(curr) + prev_level = next_level + return prev_level[0] + + +def reduce_sum_batch(tt_batch, max_tt_rank, coef=None, + name='t3f_approximate_reduce_sum_batch'): """Sum of all TT-objects in the batch with rounding after each summation. This version implements a slow-to-compile but fast-to-execute (at least on @@ -54,6 +60,7 @@ def reduce_sum_batch(tt_batch, max_tt_rank, coef=None): If coef is a matrix of shape batch_size x N, the result will be a `TensorTrainBatch` res containing N TT-object such that res[j] ~= sum_i tt_batch[i] coef[i, j] + name: string, name of the Op. Returns: If coefficients are absent or is a vector of numbers, returns @@ -70,98 +77,102 @@ def reduce_sum_batch(tt_batch, max_tt_rank, coef=None): shape = tt_batch.get_raw_shape() dtype = tt_batch.dtype - is_batch_output = False + all_tensors = tt_batch.tt_cores if coef is not None: - coef = tf.convert_to_tensor(coef) - if len(coef.get_shape()) == 1: - tt_batch = batch_ops.multiply_along_batch_dim(tt_batch, coef) - elif len(coef.get_shape()) == 2: - is_batch_output = True - output_size = coef.get_shape().as_list()[1] - # Coef is of size batch_size x N, need to duplicate the batch - # dimension xN. - if coef.shape[0] != tt_batch.batch_size: - raise ValueError('If coef is a matrix, it should be of shape ' - 'batch_size x N, got %d x %d instead ' - '(batch size is %d).' % (coef.shape[0], coef.shape[1], - tt_batch.batch_size)) - tt_batch_cores = [] + all_tensors += (coef, ) + with tf.name_scope(name, values=all_tensors): + is_batch_output = False + if coef is not None: + coef = tf.convert_to_tensor(coef) + if len(coef.get_shape()) == 1: + tt_batch = batch_ops.multiply_along_batch_dim(tt_batch, coef) + elif len(coef.get_shape()) == 2: + is_batch_output = True + output_size = coef.get_shape().as_list()[1] + # Coef is of size batch_size x N, need to duplicate the batch + # dimension xN. + if coef.shape[0] != tt_batch.batch_size: + raise ValueError('If coef is a matrix, it should be of shape ' + 'batch_size x N, got %d x %d instead ' + '(batch size is %d).' % (coef.shape[0], coef.shape[1], + tt_batch.batch_size)) + tt_batch_cores = [] + for core_idx in range(ndims): + curr_core = tt_batch.tt_cores[core_idx] + curr_shape = curr_core.get_shape().as_list() + new_shape = np.insert(curr_shape, 1, 1) + tiling = np.ones(len(new_shape)) + tiling[1] = output_size + curr_core = tf.tile(tf.reshape(curr_core, new_shape), tiling) + if core_idx == 0: + # Multiply the first TT-core by the provided coefficients. + # TODO: add t3f.utils.expands_dims_like(coef, curr_core) + shaped_coef = coef + for _ in range(len(curr_core.get_shape()) - len(coef.shape)): + shaped_coef = tf.expand_dims(shaped_coef, -1) + curr_core = curr_core * shaped_coef + # Merge the first two dimensions back into one. + raveled_shape = np.array(curr_shape).copy() + raveled_shape[0] *= output_size + curr_core = tf.reshape(curr_core, raveled_shape) + tt_batch_cores.append(curr_core) + tt_batch = TensorTrainBatch(tt_batch_cores, shape, + tt_batch.get_tt_ranks()) + + else: + raise ValueError('Coef cannot be more than 2-d.') + + if not is_batch_output: + output_size = 1 + + prev_level = tt_batch + while prev_level.batch_size > output_size: + current_level_cores = [] for core_idx in range(ndims): - curr_core = tt_batch.tt_cores[core_idx] - curr_shape = curr_core.get_shape().as_list() - new_shape = np.insert(curr_shape, 1, 1) - tiling = np.ones(len(new_shape)) - tiling[1] = output_size - curr_core = tf.tile(tf.reshape(curr_core, new_shape), tiling) - if core_idx == 0: - # Multiply the first TT-core by the provided coefficients. - # TODO: add t3f.utils.expands_dims_like(coef, curr_core) - shaped_coef = coef - for _ in range(len(curr_core.get_shape()) - len(coef.shape)): - shaped_coef = tf.expand_dims(shaped_coef, -1) - curr_core = curr_core * shaped_coef - # Merge the first two dimensions back into one. - raveled_shape = np.array(curr_shape).copy() - raveled_shape[0] *= output_size - curr_core = tf.reshape(curr_core, raveled_shape) - tt_batch_cores.append(curr_core) - tt_batch = TensorTrainBatch(tt_batch_cores, shape, - tt_batch.get_tt_ranks()) + curr_orig_core = prev_level.tt_cores[core_idx] + if is_batch_output: + # Split the first dimension into batch_size x N + unraveled_shape = curr_orig_core.get_shape().as_list() + unraveled_shape = np.array(unraveled_shape).copy() + unraveled_shape[0] /= output_size + unraveled_shape = np.insert(unraveled_shape, 1, output_size) + curr_orig_core = tf.reshape(curr_orig_core, unraveled_shape) + + a_core = curr_orig_core[::2] + b_core = curr_orig_core[1::2] + + if a_core.get_shape()[0] > b_core.get_shape()[0]: + # Odd number of elements in the batch, will have to add dummy + # TT-object with the tt-cores filled with zeros. + zeros_shape = b_core.get_shape().as_list() + zeros_shape[0] = 1 + zeros = tf.zeros(zeros_shape, dtype) + b_core = tf.concat((b_core, zeros), axis=0) + + if is_batch_output: + # Merge the first two dimensions back into one. + a_core_shape = a_core.get_shape().as_list() + a_core_shape[0] = a_core_shape[0] * a_core_shape[1] + a_core_shape = np.delete(a_core_shape, 1) + a_core = tf.reshape(a_core, a_core_shape) + b_core_shape = b_core.get_shape().as_list() + b_core_shape[0] = b_core_shape[0] * b_core_shape[1] + b_core_shape = np.delete(b_core_shape, 1) + b_core = tf.reshape(b_core, b_core_shape) + if core_idx == 0: + curr_sum_core = tf.concat((a_core, b_core), axis=right_tt_rank_dim) + elif core_idx == ndims - 1: + curr_sum_core = tf.concat((a_core, b_core), axis=left_tt_rank_dim) + else: + zeros = tf.zeros(b_core.get_shape(), dtype) + upper = tf.concat((a_core, zeros), axis=right_tt_rank_dim) + lower = tf.concat((zeros, b_core), axis=right_tt_rank_dim) + curr_sum_core = tf.concat((upper, lower), axis=left_tt_rank_dim) + current_level_cores.append(curr_sum_core) + current_level = TensorTrainBatch(current_level_cores, shape) + prev_level = decompositions.round(current_level, max_tt_rank) + if is_batch_output: + return prev_level else: - raise ValueError('Coef cannot be more than 2-d.') - - if not is_batch_output: - output_size = 1 - - prev_level = tt_batch - while prev_level.batch_size > output_size: - current_level_cores = [] - for core_idx in range(ndims): - curr_orig_core = prev_level.tt_cores[core_idx] - if is_batch_output: - # Split the first dimension into batch_size x N - unraveled_shape = curr_orig_core.get_shape().as_list() - unraveled_shape = np.array(unraveled_shape).copy() - unraveled_shape[0] /= output_size - unraveled_shape = np.insert(unraveled_shape, 1, output_size) - curr_orig_core = tf.reshape(curr_orig_core, unraveled_shape) - - a_core = curr_orig_core[::2] - b_core = curr_orig_core[1::2] - - if a_core.get_shape()[0] > b_core.get_shape()[0]: - # Odd number of elements in the batch, will have to add dummy - # TT-object with the tt-cores filled with zeros. - zeros_shape = b_core.get_shape().as_list() - zeros_shape[0] = 1 - zeros = tf.zeros(zeros_shape, dtype) - b_core = tf.concat((b_core, zeros), axis=0) - - if is_batch_output: - # Merge the first two dimensions back into one. - a_core_shape = a_core.get_shape().as_list() - a_core_shape[0] = a_core_shape[0] * a_core_shape[1] - a_core_shape = np.delete(a_core_shape, 1) - a_core = tf.reshape(a_core, a_core_shape) - b_core_shape = b_core.get_shape().as_list() - b_core_shape[0] = b_core_shape[0] * b_core_shape[1] - b_core_shape = np.delete(b_core_shape, 1) - b_core = tf.reshape(b_core, b_core_shape) - - if core_idx == 0: - curr_sum_core = tf.concat((a_core, b_core), axis=right_tt_rank_dim) - elif core_idx == ndims - 1: - curr_sum_core = tf.concat((a_core, b_core), axis=left_tt_rank_dim) - else: - zeros = tf.zeros(b_core.get_shape(), dtype) - upper = tf.concat((a_core, zeros), axis=right_tt_rank_dim) - lower = tf.concat((zeros, b_core), axis=right_tt_rank_dim) - curr_sum_core = tf.concat((upper, lower), axis=left_tt_rank_dim) - current_level_cores.append(curr_sum_core) - current_level = TensorTrainBatch(current_level_cores, shape) - prev_level = decompositions.round(current_level, max_tt_rank) - if is_batch_output: - return prev_level - else: - return prev_level[0] + return prev_level[0] diff --git a/t3f/initializers.py b/t3f/initializers.py index 6f192cc0..31a68c7a 100644 --- a/t3f/initializers.py +++ b/t3f/initializers.py @@ -69,11 +69,12 @@ def _validate_input_parameters(is_tensor, shape, **params): '1 or %d, got %d' % (shape[0].size + 1, tt_rank.size)) -def tensor_ones(shape): +def tensor_ones(shape, name='t3f_tensor_ones'): """Generate TT-tensor of the given shape with all entries equal to 1. Args: shape: array representing the shape of the future tensor + name: string, name of the Op. Returns: TensorTrain object containing a TT-tensor @@ -84,19 +85,21 @@ def tensor_ones(shape): num_dims = shape.size tt_rank = np.ones(num_dims + 1) - tt_cores = num_dims * [None] - for i in range(num_dims): - curr_core_shape = (1, shape[i], 1) - tt_cores[i] = tf.ones(curr_core_shape) + with tf.name_scope(name): + tt_cores = num_dims * [None] + for i in range(num_dims): + curr_core_shape = (1, shape[i], 1) + tt_cores[i] = tf.ones(curr_core_shape) - return TensorTrain(tt_cores, shape, tt_rank) + return TensorTrain(tt_cores, shape, tt_rank) -def tensor_zeros(shape): +def tensor_zeros(shape, name='t3f_tensor_zeros'): """Generate TT-tensor of the given shape with all entries equal to 0. Args: shape: array representing the shape of the future tensor + name: string, name of the Op. Returns: TensorTrain object containing a TT-tensor @@ -107,19 +110,21 @@ def tensor_zeros(shape): num_dims = shape.size tt_rank = np.ones(num_dims + 1) tt_cores = num_dims * [None] - for i in range(num_dims): - curr_core_shape = (1, shape[i], 1) - tt_cores[i] = tf.zeros(curr_core_shape) + with tf.name_scope(name): + for i in range(num_dims): + curr_core_shape = (1, shape[i], 1) + tt_cores[i] = tf.zeros(curr_core_shape) - return TensorTrain(tt_cores, shape, tt_rank) + return TensorTrain(tt_cores, shape, tt_rank) -def eye(shape): +def eye(shape, name='t3f_eye'): """Creates an identity TT-matrix. Args: shape: array which defines the shape of the matrix row and column - indices. + indices. + name: string, name of the Op. Returns: TensorTrain containing an identity TT-matrix of size @@ -132,16 +137,17 @@ def eye(shape): num_dims = shape.size tt_ranks = np.ones(num_dims + 1) - tt_cores = num_dims * [None] - for i in range(num_dims): - curr_core_shape = (1, shape[i], shape[i], 1) - tt_cores[i] = tf.reshape(tf.eye(shape[i]), curr_core_shape) + with tf.name_scope(name): + tt_cores = num_dims * [None] + for i in range(num_dims): + curr_core_shape = (1, shape[i], shape[i], 1) + tt_cores[i] = tf.reshape(tf.eye(shape[i]), curr_core_shape) - true_shape = np.vstack([shape, shape]) - return TensorTrain(tt_cores, true_shape, tt_ranks) + true_shape = np.vstack([shape, shape]) + return TensorTrain(tt_cores, true_shape, tt_ranks) -def matrix_ones(shape): +def matrix_ones(shape, name='t3f_matrix_ones'): """Generate a TT-matrix of the given shape with each entry equal to 1. Args: @@ -153,6 +159,7 @@ def matrix_ones(shape): and matrix_ones([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. + name: string, name of the Op. Returns: TensorTrain containing a TT-matrix of size @@ -173,17 +180,16 @@ def matrix_ones(shape): num_dims = shape[0].size tt_rank = np.ones(shape[0].size + 1) - # TODO: variable (name?) scope. - - tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (1, shape[0][i], shape[1][i], 1) - tt_cores[i] = tf.ones(curr_core_shape) + with tf.name_scope(name): + tt_cores = [None] * num_dims + for i in range(num_dims): + curr_core_shape = (1, shape[0][i], shape[1][i], 1) + tt_cores[i] = tf.ones(curr_core_shape) - return TensorTrain(tt_cores, shape, tt_rank) + return TensorTrain(tt_cores, shape, tt_rank) -def matrix_zeros(shape): +def matrix_zeros(shape, name='t3f_matrix_zeros'): """Generate a TT-matrix of the given shape with each entry equal to 0. Args: @@ -195,6 +201,7 @@ def matrix_zeros(shape): and matrix_zeros([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. + name: string, name of the Op. Returns: TensorTrain containing a TT-matrix of size @@ -214,17 +221,17 @@ def matrix_zeros(shape): num_dims = shape[0].size tt_rank = np.ones(shape[0].size + 1) - # TODO: variable (name?) scope. + with tf.name_scope(name): + tt_cores = [None] * num_dims + for i in range(num_dims): + curr_core_shape = (1, shape[0][i], shape[1][i], 1) + tt_cores[i] = tf.zeros(curr_core_shape) - tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (1, shape[0][i], shape[1][i], 1) - tt_cores[i] = tf.zeros(curr_core_shape) - - return TensorTrain(tt_cores, shape, tt_rank) + return TensorTrain(tt_cores, shape, tt_rank) -def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): +def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1., + name='t3f_tensor_with_random_cores'): """Generate a TT-tensor of the given shape with N(mean, stddev^2) cores. Args: @@ -234,6 +241,7 @@ def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): initializing TT-cores. stddev: a number, the standard deviation of the normal distribution used for initializing TT-cores. + name: string, name of the Op. Returns: TensorTrain containing a TT-tensor @@ -251,17 +259,18 @@ def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): tt_rank = np.append(tt_rank, 1) tt_rank = tt_rank.astype(int) - # TODO: variable (name?) scope. tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (tt_rank[i], shape[i], tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + with tf.name_scope(name): + for i in range(num_dims): + curr_core_shape = (tt_rank[i], shape[i], tt_rank[i + 1]) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) - return TensorTrain(tt_cores, shape, tt_rank) + return TensorTrain(tt_cores, shape, tt_rank) def tensor_batch_with_random_cores(shape, tt_rank=2, batch_size=1, - mean=0., stddev=1.): + mean=0., stddev=1., + name='t3f_tensor_batch_with_random_cores'): """Generate a batch of TT-tensors of given shape with N(mean, stddev^2) cores. Args: @@ -272,6 +281,7 @@ def tensor_batch_with_random_cores(shape, tt_rank=2, batch_size=1, initializing TT-cores. stddev: a number, the standard deviation of the normal distribution used for initializing TT-cores. + name: string, name of the Op. Returns: TensorTrainBatch containing TT-tensors @@ -289,16 +299,17 @@ def tensor_batch_with_random_cores(shape, tt_rank=2, batch_size=1, tt_rank = np.insert(tt_rank, 0, 1) tt_rank = np.append(tt_rank, 1) tt_rank = tt_rank.astype(int) - # TODO: variable (name?) scope. tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (batch_size, tt_rank[i], shape[i], tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + with tf.name_scope(name): + for i in range(num_dims): + curr_core_shape = (batch_size, tt_rank[i], shape[i], tt_rank[i + 1]) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) - return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) + return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) -def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): +def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1., + name='t3f_matrix_with_random_cores'): """Generate a TT-matrix of given shape with N(mean, stddev^2) cores. Args: @@ -315,6 +326,7 @@ def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): initializing TT-cores. stddev: a number, the standard deviation of the normal distribution used for initializing TT-cores. + name: string, name of the Op. Returns: TensorTrain containing a TT-matrix of size @@ -339,18 +351,19 @@ def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): tt_rank = np.concatenate([[1], tt_rank, [1]]) tt_rank = tt_rank.astype(int) - # TODO: variable (name?) scope. tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (tt_rank[i], shape[0][i], shape[1][i], - tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + with tf.name_scope(name): + for i in range(num_dims): + curr_core_shape = (tt_rank[i], shape[0][i], shape[1][i], + tt_rank[i + 1]) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) - return TensorTrain(tt_cores, shape, tt_rank) + return TensorTrain(tt_cores, shape, tt_rank) def matrix_batch_with_random_cores(shape, tt_rank=2, batch_size=1, - mean=0., stddev=1.): + mean=0., stddev=1., + name='t3f_matrix_batch_with_random_cores'): """Generate a batch of TT-matrices of given shape with N(mean, stddev^2) cores. Args: @@ -369,6 +382,7 @@ def matrix_batch_with_random_cores(shape, tt_rank=2, batch_size=1, initializing TT-cores. stddev: a number, the standard deviation of the normal distribution used for initializing TT-cores. + name: string, name of the Op. Returns: TensorTrainBatch containing a batch of TT-matrices of size @@ -393,17 +407,17 @@ def matrix_batch_with_random_cores(shape, tt_rank=2, batch_size=1, tt_rank = np.concatenate([[1], tt_rank, [1]]) shape = shape.astype(int) tt_rank = tt_rank.astype(int) - # TODO: variable (name?) scope. tt_cores = [None] * num_dims - for i in range(num_dims): - curr_core_shape = (batch_size, tt_rank[i], shape[0][i], shape[1][i], - tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + with tf.name_scope(name): + for i in range(num_dims): + curr_core_shape = (batch_size, tt_rank[i], shape[0][i], shape[1][i], + tt_rank[i + 1]) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) - return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) + return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) -def ones_like(tt): +def ones_like(tt, name='t3f_ones_like'): """Constructs t3f.ones with the shape of `tt`. In the case when `tt` is TensorTrainBatch constructs t3f.ones with the shape @@ -411,6 +425,7 @@ def ones_like(tt): Args: tt: TensorTrain object + name: string, name of the Op. Returns: TensorTrain object of the same shape as `tt` but with all entries equal to @@ -421,13 +436,14 @@ def ones_like(tt): raise ValueError("`tt` has to be a Tensor Train object") else: shape = shapes.lazy_raw_shape(tt) - if tt.is_tt_matrix(): - return matrix_ones(shape) - else: - return tensor_ones(shape[0, :]) + with tf.name_scope(name, values=tt.tt_cores): + if tt.is_tt_matrix(): + return matrix_ones(shape) + else: + return tensor_ones(shape[0, :]) -def zeros_like(tt): +def zeros_like(tt, name='t3f_zeros_like'): """Constructs t3f.zeros with the shape of `tt`. In the case when `tt` is a TensorTrainBatch constructs t3f.zeros with @@ -435,6 +451,7 @@ def zeros_like(tt): Args: tt: TensorTrain object + name: string, name of the Op. Returns: TensorTrain object of the same shape as `tt` but with all entries equal to @@ -445,13 +462,15 @@ def zeros_like(tt): raise ValueError("`tt` has to be a Tensor Train object") else: shape = shapes.lazy_raw_shape(tt) - if tt.is_tt_matrix(): - return matrix_zeros(shape) - else: - return tensor_zeros(shape[0, :]) + with tf.name_scope(name, values=tt.tt_cores): + if tt.is_tt_matrix(): + return matrix_zeros(shape) + else: + return tensor_zeros(shape[0, :]) -def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): +def random_tensor(shape, tt_rank=2, mean=0., stddev=1., + name='t3f_random_tensor'): """Generate a random TT-tensor of the given shape with given mean and stddev. Entries of the generated tensor (in the full format) will be iid and satisfy @@ -470,6 +489,7 @@ def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): mean: a number, the desired mean for the distribution of entries. stddev: a number, the desired standard deviation for the distribution of entries. + name: string, name of the Op. Returns: TensorTrain containing a TT-tensor @@ -493,7 +513,8 @@ def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): cr_exponent = -1.0 / (2 * num_dims) var = np.prod(tt_rank ** cr_exponent) core_stddev = stddev ** (1.0 / num_dims) * var - tt = tensor_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) + with tf.name_scope(name): + tt = tensor_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) if np.abs(mean) < 1e-8: return tt @@ -501,7 +522,8 @@ def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): raise NotImplementedError('non-zero mean is not supported yet') -def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): +def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1., + name='t3f_random_tensor_batch'): """Generate a batch of TT-tensors with given shape, mean and stddev. Entries of the generated tensors (in the full format) will be iid and satisfy @@ -521,6 +543,7 @@ def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): mean: a number, the desired mean for the distribution of entries. stddev: a number, the desired standard deviation for the distribution of entries. + name: string, name of the Op. Returns: TensorTrainBatch containing TT-tensors. @@ -541,8 +564,9 @@ def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): cr_exponent = -1.0 / (2 * num_dims) var = np.prod(tt_rank ** cr_exponent) cr_stddev = stddev ** (1.0 / num_dims) * var - tt = tensor_batch_with_random_cores(shape, tt_rank=tt_rank, stddev=cr_stddev, - batch_size=batch_size) + with tf.name_scope(name): + tt = tensor_batch_with_random_cores(shape, tt_rank=tt_rank, + stddev=cr_stddev, batch_size=batch_size) if np.abs(mean) < 1e-8: return tt @@ -550,7 +574,8 @@ def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): raise NotImplementedError('non-zero mean is not supported yet') -def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): +def random_matrix(shape, tt_rank=2, mean=0., stddev=1., + name='t3f_random_matrix'): """Generate a random TT-matrix of the given shape with given mean and stddev. Entries of the generated matrix (in the full format) will be iid and satisfy @@ -575,6 +600,7 @@ def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): mean: a number, the desired mean for the distribution of entries. stddev: a number, the desired standard deviation for the distribution of entries. + name: string, name of the Op. Returns: TensorTrain containing a TT-matrix of size @@ -609,7 +635,8 @@ def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): cr_exponent = -1.0 / (2 * num_dims) var = np.prod(tt_rank ** cr_exponent) core_stddev = stddev ** (1.0 / num_dims) * var - tt = matrix_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) + with tf.name_scope(name): + tt = matrix_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) if np.abs(mean) < 1e-8: return tt @@ -617,7 +644,8 @@ def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): raise NotImplementedError('non-zero mean is not supported yet') -def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): +def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1., + name='t3f_random_matrix_batch'): """Generate a batch of TT-matrices with given shape, mean and stddev. Entries of the generated matrices (in the full format) will be iid and @@ -643,6 +671,7 @@ def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): mean: a number, the desired mean for the distribution of entries. stddev: a number, the desired standard deviation for the distribution of entries. + name: string, name of the Op. Returns: TensorTrainBatch containing a batch of TT-matrices of size @@ -671,9 +700,10 @@ def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): cr_exponent = -1.0 / (2 * num_dims) var = np.prod(tt_rank ** cr_exponent) core_stddev = stddev ** (1.0 / num_dims) * var - tt = matrix_batch_with_random_cores(shape, tt_rank=tt_rank, - stddev=core_stddev, - batch_size=batch_size) + with tf.name_scope(name): + tt = matrix_batch_with_random_cores(shape, tt_rank=tt_rank, + stddev=core_stddev, + batch_size=batch_size) if np.abs(mean) < 1e-8: return tt @@ -681,7 +711,7 @@ def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): raise NotImplementedError('non-zero mean is not supported yet') -def glorot_initializer(shape, tt_rank=2): +def glorot_initializer(shape, tt_rank=2, name='t3f_glorot_initializer'): """Constructs a random TT matrix with entrywise variance 2.0 / (n_in + n_out) Args: @@ -694,6 +724,7 @@ def glorot_initializer(shape, tt_rank=2): glorot_initializer([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. tt_rank: a number or a (d+1)-element array with ranks. + name: string, name of the Op. Returns: TensorTrain containing a TT-matrix of size @@ -715,10 +746,11 @@ def glorot_initializer(shape, tt_rank=2): n_out = np.prod(shape[1]) lamb = 2.0 / (n_in + n_out) - return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) + with tf.name_scope(name): + return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) -def he_initializer(shape, tt_rank=2): +def he_initializer(shape, tt_rank=2, name='t3f_he_initializer'): """Constructs a random TT matrix with entrywise variance 2.0 / n_in Args: @@ -731,6 +763,7 @@ def he_initializer(shape, tt_rank=2): he_initializer([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. tt_rank: a number or a (d+1)-element array with ranks. + name: string, name of the Op. Returns: TensorTrain containing a TT-matrix of size @@ -751,10 +784,11 @@ def he_initializer(shape, tt_rank=2): n_in = np.prod(shape[0]) lamb = 2.0 / n_in - return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) + with tf.name_scope(name): + return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) -def lecun_initializer(shape, tt_rank=2): +def lecun_initializer(shape, tt_rank=2, name='t3f_lecun_initializer'): """Constructs a random TT matrix with entrywise variance 1.0 / n_in Args: @@ -767,6 +801,7 @@ def lecun_initializer(shape, tt_rank=2): lecun_initializer([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. tt_rank: a number or a (d+1)-element array with ranks. + name: string, name of the Op. Returns: TensorTrain containing a TT-matrix of size @@ -786,4 +821,5 @@ def lecun_initializer(shape, tt_rank=2): _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) n_in = np.prod(shape[0]) lamb = 1.0 / n_in - return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) + with tf.name_scope(name): + return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) From 8a27d0b0aabc6c11d2075af3c232e5bc2ce39769 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 6 May 2018 13:31:19 +0300 Subject: [PATCH 035/233] Remove outdated todos --- t3f/ops.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index 7af9bf11..25c59653 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -149,7 +149,6 @@ def tt_tt_matmul(tt_matrix_a, tt_matrix_b): einsum_str = '{}aijb,{}cjkd->{}acikbd'.format(a_batch_str, b_batch_str, res_batch_str) result_cores = [] - # TODO: name the operation and the resulting tensor. a_shape = shapes.lazy_raw_shape(tt_matrix_a) a_ranks = shapes.lazy_tt_ranks(tt_matrix_a) b_shape = shapes.lazy_raw_shape(tt_matrix_b) @@ -387,7 +386,6 @@ def tt_tt_flat_inner(tt_a, tt_b): # if both arguments are TT-tensors, then it is # res = tf.einsum('aib,cid->bd', a_core, b_core) res = tf.einsum(init_einsum_str, a_core, b_core) - # TODO: name the operation and the resulting tensor. einsum_str = '{3}ac,{1}a{0}b,{2}c{0}d->{3}bd'.format(axes_str, a_batch_str, b_batch_str, From 400bb15417b3425df16970fc4afa0dd5e9e24b90 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Thu, 10 May 2018 23:42:42 +0300 Subject: [PATCH 036/233] init auto diff code --- t3f/Auto diff.ipynb | 205 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 t3f/Auto diff.ipynb diff --git a/t3f/Auto diff.ipynb b/t3f/Auto diff.ipynb new file mode 100644 index 00000000..0c4c1043 --- /dev/null +++ b/t3f/Auto diff.ipynb @@ -0,0 +1,205 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import t3f\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# tf.enable_eager_execution()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "w = t3f.random_tensor((3, 4, 5))\n", + "w = t3f.orthogonalize_tt_cores(w, left_to_right=False)\n", + "w = t3f.get_variable('w', initializer=w)\n", + "x = t3f.random_tensor((3, 4, 5))\n", + "x = t3f.get_variable('x', initializer=x)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def tangent_space_to_deltas(tt):\n", + " if tt.projection_on is None:\n", + " raise ValueError('tt argument is supposed to be a projection, but it lacks projection_on field')\n", + " num_dims = tt.ndims()\n", + " deltas = [None] * num_dims\n", + " for i in range(1, num_dims - 1):\n", + " r1, _, r2 = tt.tt_cores[i].get_shape().as_list()\n", + " if int(r1 / 2) != r1 / 2:\n", + " raise ValueError('tt argument is supposed to be a projection, but its ranks are not even.')\n", + " deltas[i] = tt.tt_cores[i][int(r1 / 2):, :, :int(r2 / 2)]\n", + " _, _, r = tt.tt_cores[0].get_shape().as_list()\n", + " deltas[0] = tt.tt_cores[0][:, :, :int(r / 2)]\n", + " r, _, _ = tt.tt_cores[num_dims - 1].get_shape().as_list()\n", + " deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][int(r / 2):, :, :]\n", + " return deltas\n", + "\n", + "def left_q(X, i):\n", + " \"\"\"Compute the orthogonal matrix Q_{\\leq i} as defined in [1].\"\"\"\n", + " if i < 0:\n", + " return np.ones([1, 1], dtype=np.float32)\n", + " answ = np.ones([1, 1])\n", + " for dim in range(i + 1):\n", + " answ = np.tensordot(answ, sess.run(X.tt_cores[dim]), 1)\n", + " answ = np.reshape(answ, (-1, answ.shape[-1]))\n", + " return answ.astype(np.float32)\n", + "\n", + "def right_q(X, i):\n", + " \"\"\"Compute the orthogonal matrix Q_{\\geq i} as defined in [1].\"\"\"\n", + " if i > X.ndims() - 1:\n", + " return np.ones([1, 1], dtype=np.float32)\n", + " answ = np.ones([1, 1])\n", + " for dim in range(X.ndims() - 1, i - 1, -1):\n", + " answ = np.tensordot(sess.run(X.tt_cores[dim]), answ, 1)\n", + " answ = np.reshape(answ, (answ.shape[0], -1))\n", + " return answ.T.astype(np.float32)\n", + "\n", + "def deltas_to_tangent_space(deltas, tt, left, right):\n", + " cores = []\n", + " dtype = deltas[0].dtype\n", + " num_dims = left.ndims()\n", + " left_tangent_tt_ranks = t3f.shapes.lazy_tt_ranks(left)\n", + " right_tangent_tt_ranks = t3f.shapes.lazy_tt_ranks(left)\n", + " raw_shape = t3f.shapes.lazy_raw_shape(left)\n", + " right_rank_dim = left.right_tt_rank_dim\n", + " left_rank_dim = left.left_tt_rank_dim\n", + " for i in range(num_dims):\n", + " left_tt_core = left.tt_cores[i]\n", + " right_tt_core = right.tt_cores[i]\n", + "\n", + " if i == 0:\n", + " tangent_core = tf.concat((deltas[i], left_tt_core),\n", + " axis=right_rank_dim)\n", + " elif i == num_dims - 1:\n", + " tangent_core = tf.concat((right_tt_core, deltas[i]),\n", + " axis=left_rank_dim)\n", + " else:\n", + " rank_1 = right_tangent_tt_ranks[i]\n", + " rank_2 = left_tangent_tt_ranks[i + 1]\n", + " mode_size_n = raw_shape[0][i]\n", + "# mode_size_m = raw_shape[1][i]\n", + " shape = [rank_1, mode_size_n, rank_2]\n", + " zeros = tf.zeros(shape, dtype)\n", + " upper = tf.concat((right_tt_core, zeros), axis=right_rank_dim)\n", + " lower = tf.concat((deltas[i], left_tt_core), axis=right_rank_dim)\n", + " tangent_core = tf.concat((upper, lower), axis=left_rank_dim)\n", + " cores.append(tangent_core)\n", + " tangent = t3f.TensorTrain(cores)\n", + " tangent.projection_on = tt\n", + " return tangent\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "def riemannian_grad(func, w):\n", + " # TODO: can do cheaper with just two orthogonalizations.\n", + " w_projection = t3f.project(w, w)\n", + " h = func(w_projection)\n", + " cores_grad = tf.gradients(h, w_projection.tt_cores)\n", + " deltas = []\n", + " left = t3f.orthogonalize_tt_cores(w)\n", + " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", + " for i in range(w.ndims()):\n", + " r1, n, r2 = left.tt_cores[i].shape.as_list()\n", + " q = tf.reshape(left.tt_cores[i], (-1, r2))\n", + " if i == 0:\n", + " curr_grad = cores_grad[i][:, :, :r2]\n", + " elif i == w.ndims() - 1:\n", + " curr_grad = cores_grad[i][r1:, :, :]\n", + " else:\n", + " curr_grad = cores_grad[i][r1:, :, :r2]\n", + " if i < w.ndims() - 1:\n", + " proj = (tf.eye(r1 * n) - q @ tf.transpose(q))\n", + " delta = proj @ tf.reshape(curr_grad, (-1, r2))\n", + " delta = tf.reshape(delta, w.tt_cores[i].shape)\n", + " else:\n", + " delta = curr_grad\n", + " deltas.append(delta)\n", + " return deltas_to_tangent_space(deltas, w, left, right)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "sess = tf.Session()\n", + "sess.run(tf.global_variables_initializer())" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "func = lambda w: 0.5 * t3f.flat_inner(x, w) ** 2\n", + "desired, actual = sess.run([t3f.full(t3f.flat_inner(x, w) * t3f.project(x, w)), t3f.full(riemannian_grad(func, w))])\n", + "np.testing.assert_allclose(desired, actual, rtol=1e-5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 5b1ab6887ac4b8fcdfb9d29c850ebe95ffb97d7b Mon Sep 17 00:00:00 2001 From: bihaqo Date: Fri, 11 May 2018 22:31:51 +0300 Subject: [PATCH 037/233] speed it up by avoiding explicit projection --- t3f/Auto diff.ipynb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/t3f/Auto diff.ipynb b/t3f/Auto diff.ipynb index 0c4c1043..c04c6e9e 100644 --- a/t3f/Auto diff.ipynb +++ b/t3f/Auto diff.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -30,7 +30,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -43,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -121,13 +121,15 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "def riemannian_grad(func, w):\n", - " # TODO: can do cheaper with just two orthogonalizations.\n", - " w_projection = t3f.project(w, w)\n", + " left = t3f.orthogonalize_tt_cores(w)\n", + " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", + " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", + " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", " h = func(w_projection)\n", " cores_grad = tf.gradients(h, w_projection.tt_cores)\n", " deltas = []\n", @@ -154,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -164,7 +166,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ From d2f591501d3c36523f88e5b268f9b2428eecd10c Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 13 May 2018 00:53:07 +0300 Subject: [PATCH 038/233] actually dont need stop grads --- t3f/Auto diff.ipynb | 252 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 211 insertions(+), 41 deletions(-) diff --git a/t3f/Auto diff.ipynb b/t3f/Auto diff.ipynb index c04c6e9e..079cb406 100644 --- a/t3f/Auto diff.ipynb +++ b/t3f/Auto diff.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 4, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -30,20 +30,43 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ - "w = t3f.random_tensor((3, 4, 5))\n", - "w = t3f.orthogonalize_tt_cores(w, left_to_right=False)\n", + "w = t3f.random_matrix(([10] * 3, None))\n", "w = t3f.get_variable('w', initializer=w)\n", - "x = t3f.random_tensor((3, 4, 5))\n", - "x = t3f.get_variable('x', initializer=x)" + "A = t3f.random_matrix(([10] * 3, [10] * 3))\n", + "A = t3f.get_variable('A', initializer=A)\n", + "z = t3f.random_matrix(([10] * 3, None))\n", + "z = t3f.get_variable('z', initializer=z)\n", + "# x = t3f.random_matrix(([10] * 10, None), tt_rank=100)\n", + "# x = t3f.get_variable('x', initializer=x)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "w.is_tt_matrix()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -53,15 +76,26 @@ " raise ValueError('tt argument is supposed to be a projection, but it lacks projection_on field')\n", " num_dims = tt.ndims()\n", " deltas = [None] * num_dims\n", - " for i in range(1, num_dims - 1):\n", - " r1, _, r2 = tt.tt_cores[i].get_shape().as_list()\n", - " if int(r1 / 2) != r1 / 2:\n", - " raise ValueError('tt argument is supposed to be a projection, but its ranks are not even.')\n", - " deltas[i] = tt.tt_cores[i][int(r1 / 2):, :, :int(r2 / 2)]\n", - " _, _, r = tt.tt_cores[0].get_shape().as_list()\n", - " deltas[0] = tt.tt_cores[0][:, :, :int(r / 2)]\n", - " r, _, _ = tt.tt_cores[num_dims - 1].get_shape().as_list()\n", - " deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][int(r / 2):, :, :]\n", + " if tt.is_tt_matrix():\n", + " for i in range(1, num_dims - 1):\n", + " r1, _, _, r2 = tt.tt_cores[i].get_shape().as_list()\n", + " if int(r1 / 2) != r1 / 2:\n", + " raise ValueError('tt argument is supposed to be a projection, but its ranks are not even.')\n", + " deltas[i] = tt.tt_cores[i][int(r1 / 2):, :, :, :int(r2 / 2)]\n", + " _, _, _, r = tt.tt_cores[0].get_shape().as_list()\n", + " deltas[0] = tt.tt_cores[0][:, :, :, :int(r / 2)]\n", + " r, _, _, _ = tt.tt_cores[num_dims - 1].get_shape().as_list()\n", + " deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][int(r / 2):, :, :, :]\n", + " else:\n", + " for i in range(1, num_dims - 1):\n", + " r1, _, r2 = tt.tt_cores[i].get_shape().as_list()\n", + " if int(r1 / 2) != r1 / 2:\n", + " raise ValueError('tt argument is supposed to be a projection, but its ranks are not even.')\n", + " deltas[i] = tt.tt_cores[i][int(r1 / 2):, :, :int(r2 / 2)]\n", + " _, _, r = tt.tt_cores[0].get_shape().as_list()\n", + " deltas[0] = tt.tt_cores[0][:, :, :int(r / 2)]\n", + " r, _, _ = tt.tt_cores[num_dims - 1].get_shape().as_list()\n", + " deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][int(r / 2):, :, :]\n", " return deltas\n", "\n", "def left_q(X, i):\n", @@ -106,9 +140,13 @@ " else:\n", " rank_1 = right_tangent_tt_ranks[i]\n", " rank_2 = left_tangent_tt_ranks[i + 1]\n", - " mode_size_n = raw_shape[0][i]\n", - "# mode_size_m = raw_shape[1][i]\n", - " shape = [rank_1, mode_size_n, rank_2]\n", + " if tt.is_tt_matrix():\n", + " mode_size_n = raw_shape[0][i]\n", + " mode_size_m = raw_shape[1][i]\n", + " shape = [rank_1, mode_size_n, mode_size_m, rank_2]\n", + " else:\n", + " mode_size_n = raw_shape[0][i]\n", + " shape = [rank_1, mode_size_n, rank_2]\n", " zeros = tf.zeros(shape, dtype)\n", " upper = tf.concat((right_tt_core, zeros), axis=right_rank_dim)\n", " lower = tf.concat((deltas[i], left_tt_core), axis=right_rank_dim)\n", @@ -121,33 +159,46 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "def riemannian_grad(func, w):\n", - " left = t3f.orthogonalize_tt_cores(w)\n", - " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", - " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", - " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", + "def riemannian_grad(func, w, convert_to_projection=True, left=None, right=None):\n", + " if left is None or right is None:\n", + " left = t3f.orthogonalize_tt_cores(w)\n", + " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", + " if convert_to_projection:\n", + " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", + " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", + " else:\n", + " w_projection = w\n", " h = func(w_projection)\n", " cores_grad = tf.gradients(h, w_projection.tt_cores)\n", " deltas = []\n", - " left = t3f.orthogonalize_tt_cores(w)\n", - " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", " for i in range(w.ndims()):\n", - " r1, n, r2 = left.tt_cores[i].shape.as_list()\n", + " if w.is_tt_matrix():\n", + " r1, n, m, r2 = left.tt_cores[i].shape.as_list()\n", + " else:\n", + " r1, n, r2 = left.tt_cores[i].shape.as_list()\n", " q = tf.reshape(left.tt_cores[i], (-1, r2))\n", - " if i == 0:\n", - " curr_grad = cores_grad[i][:, :, :r2]\n", - " elif i == w.ndims() - 1:\n", - " curr_grad = cores_grad[i][r1:, :, :]\n", + " if w.is_tt_matrix():\n", + " if i == 0:\n", + " curr_grad = cores_grad[i][:, :, :, :r2]\n", + " elif i == w.ndims() - 1:\n", + " curr_grad = cores_grad[i][r1:, :, :, :]\n", + " else:\n", + " curr_grad = cores_grad[i][r1:, :, :, :r2]\n", " else:\n", - " curr_grad = cores_grad[i][r1:, :, :r2]\n", + " if i == 0:\n", + " curr_grad = cores_grad[i][:, :, :r2]\n", + " elif i == w.ndims() - 1:\n", + " curr_grad = cores_grad[i][r1:, :, :]\n", + " else:\n", + " curr_grad = cores_grad[i][r1:, :, :r2]\n", " if i < w.ndims() - 1:\n", " proj = (tf.eye(r1 * n) - q @ tf.transpose(q))\n", " delta = proj @ tf.reshape(curr_grad, (-1, r2))\n", - " delta = tf.reshape(delta, w.tt_cores[i].shape)\n", + " delta = tf.reshape(delta, left.tt_cores[i].shape)\n", " else:\n", " delta = curr_grad\n", " deltas.append(delta)\n", @@ -156,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -166,13 +217,132 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "func = lambda w: t3f.quadratic_form(A, w, w)\n", + "# desired, actual = sess.run([t3f.full(t3f.flat_inner(x, w) * t3f.project(x, w)), t3f.full(riemannian_grad(func, w))])\n", + "# np.testing.assert_allclose(desired, actual, rtol=1e-5)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "op1 = riemannian_grad(func, w).op\n", + "op2 = (t3f.project_matmul(t3f.expand_batch_dim(w), w, A)).op" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "115 µs ± 7.24 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" + ] + } + ], + "source": [ + "%timeit sess.run(op1)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "106 µs ± 2.58 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" + ] + } + ], + "source": [ + "%timeit sess.run(op2)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def stop_gradient(tt):\n", + " new_tt_cores = [tf.stop_gradient(c) for c in tt.tt_cores]\n", + " return t3f.TensorTrain(new_tt_cores)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "desired, actual = sess.run([t3f.full(t3f.project(2 * w, w)), t3f.full(riemannian_grad(func, w))])\n", + "np.testing.assert_allclose(desired, actual, rtol=1e-3)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "w_projected = t3f.project(w, w)\n", + "desired, actual = sess.run([t3f.full(t3f.project(2 * w, w)), t3f.full(riemannian_grad(func, w))])\n", + "np.testing.assert_allclose(desired, actual, rtol=1e-3)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def hessian_by_vector(f, w, vector):\n", + " left = t3f.orthogonalize_tt_cores(w)\n", + " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", + " vector_projected = t3f.project(vector, w)\n", + " vector_projected = t3f.expand_batch_dim(vector_projected)\n", + " vector_projected.projection_on = w\n", + " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", + " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", + " def new_f(new_w):\n", + " grad = riemannian_grad(f, w_projection, convert_to_projection=False, left=left, right=right)\n", + " grad = t3f.expand_batch_dim(grad)\n", + " # TODO: durty hack.\n", + " grad.projection_on = w\n", + " return t3f.pairwise_flat_inner_projected(grad, vector_projected)[0, 0]\n", + " return riemannian_grad(new_f, w_projection, convert_to_projection=False, left=left, right=right)" + ] + }, + { + "cell_type": "code", + "execution_count": 164, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ - "func = lambda w: 0.5 * t3f.flat_inner(x, w) ** 2\n", - "desired, actual = sess.run([t3f.full(t3f.flat_inner(x, w) * t3f.project(x, w)), t3f.full(riemannian_grad(func, w))])\n", - "np.testing.assert_allclose(desired, actual, rtol=1e-5)" + "func = lambda w: t3f.quadratic_form(A, w, w)\n", + "desired = t3f.project(t3f.matmul(t3f.transpose(A) + A, t3f.project(z, w)), w)\n", + "actual = hessian_by_vector(func, w, z)\n", + "desired, actual = sess.run([t3f.full(desired), t3f.full(actual)])\n", + "np.testing.assert_allclose(desired, actual, rtol=1e-4)" ] }, { From 534a1d86a4f2cb5da984199a30b8f058b0c1befe Mon Sep 17 00:00:00 2001 From: bihaqo Date: Sun, 13 May 2018 14:24:06 +0300 Subject: [PATCH 039/233] simplify a bit more --- t3f/Auto diff.ipynb | 116 +++++++++++++++----------------------------- 1 file changed, 40 insertions(+), 76 deletions(-) diff --git a/t3f/Auto diff.ipynb b/t3f/Auto diff.ipynb index 079cb406..2404edb7 100644 --- a/t3f/Auto diff.ipynb +++ b/t3f/Auto diff.ipynb @@ -40,8 +40,8 @@ "A = t3f.get_variable('A', initializer=A)\n", "z = t3f.random_matrix(([10] * 3, None))\n", "z = t3f.get_variable('z', initializer=z)\n", - "# x = t3f.random_matrix(([10] * 10, None), tt_rank=100)\n", - "# x = t3f.get_variable('x', initializer=x)" + "x = t3f.random_matrix(([10] * 3, None), tt_rank=100)\n", + "x = t3f.get_variable('x', initializer=x)" ] }, { @@ -159,19 +159,11 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ - "def riemannian_grad(func, w, convert_to_projection=True, left=None, right=None):\n", - " if left is None or right is None:\n", - " left = t3f.orthogonalize_tt_cores(w)\n", - " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", - " if convert_to_projection:\n", - " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", - " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", - " else:\n", - " w_projection = w\n", + "def _riemannian_grad(func, w, w_projection, left, right):\n", " h = func(w_projection)\n", " cores_grad = tf.gradients(h, w_projection.tt_cores)\n", " deltas = []\n", @@ -202,12 +194,32 @@ " else:\n", " delta = curr_grad\n", " deltas.append(delta)\n", - " return deltas_to_tangent_space(deltas, w, left, right)" + " return deltas_to_tangent_space(deltas, w, left, right)\n", + "def riemannian_grad(func, w):\n", + " left = t3f.orthogonalize_tt_cores(w)\n", + " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", + " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", + " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", + " return _riemannian_grad(func, w, w_projection, left, right)\n", + "\n", + "def hessian_by_vector(f, w, vector):\n", + " left = t3f.orthogonalize_tt_cores(w)\n", + " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", + " vector_projected = t3f.project(vector, w)\n", + " vector_projected = t3f.expand_batch_dim(vector_projected)\n", + " vector_projected.projection_on = w\n", + " def new_f(new_w):\n", + " grad = _riemannian_grad(f, w, new_w, left, right)\n", + " grad = t3f.expand_batch_dim(grad)\n", + " # TODO: durty hack.\n", + " grad.projection_on = w\n", + " return t3f.pairwise_flat_inner_projected(grad, vector_projected)[0, 0]\n", + " return riemannian_grad(new_f, w)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -217,13 +229,19 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ + "func = lambda w: 0.5 * t3f.flat_inner(x, w)**2\n", + "desired, actual = sess.run([t3f.full(t3f.flat_inner(x, w) * t3f.project(x, w)), t3f.full(riemannian_grad(func, w))])\n", + "np.testing.assert_allclose(desired, actual, rtol=1e-3)\n", + "\n", "func = lambda w: t3f.quadratic_form(A, w, w)\n", - "# desired, actual = sess.run([t3f.full(t3f.flat_inner(x, w) * t3f.project(x, w)), t3f.full(riemannian_grad(func, w))])\n", - "# np.testing.assert_allclose(desired, actual, rtol=1e-5)" + "desired = t3f.project(t3f.matmul(t3f.transpose(A) + A, t3f.project(z, w)), w)\n", + "actual = hessian_by_vector(func, w, z)\n", + "desired, actual = sess.run([t3f.full(desired), t3f.full(actual)])\n", + "np.testing.assert_allclose(desired, actual, rtol=1e-2)" ] }, { @@ -272,78 +290,24 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 25, "metadata": {}, "outputs": [], - "source": [ - "def stop_gradient(tt):\n", - " new_tt_cores = [tf.stop_gradient(c) for c in tt.tt_cores]\n", - " return t3f.TensorTrain(new_tt_cores)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "desired, actual = sess.run([t3f.full(t3f.project(2 * w, w)), t3f.full(riemannian_grad(func, w))])\n", - "np.testing.assert_allclose(desired, actual, rtol=1e-3)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [], - "source": [ - "w_projected = t3f.project(w, w)\n", - "desired, actual = sess.run([t3f.full(t3f.project(2 * w, w)), t3f.full(riemannian_grad(func, w))])\n", - "np.testing.assert_allclose(desired, actual, rtol=1e-3)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "def hessian_by_vector(f, w, vector):\n", - " left = t3f.orthogonalize_tt_cores(w)\n", - " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", - " vector_projected = t3f.project(vector, w)\n", - " vector_projected = t3f.expand_batch_dim(vector_projected)\n", - " vector_projected.projection_on = w\n", - " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", - " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", - " def new_f(new_w):\n", - " grad = riemannian_grad(f, w_projection, convert_to_projection=False, left=left, right=right)\n", - " grad = t3f.expand_batch_dim(grad)\n", - " # TODO: durty hack.\n", - " grad.projection_on = w\n", - " return t3f.pairwise_flat_inner_projected(grad, vector_projected)[0, 0]\n", - " return riemannian_grad(new_f, w_projection, convert_to_projection=False, left=left, right=right)" - ] + "source": [] }, { "cell_type": "code", - "execution_count": 164, + "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 26, "metadata": {}, "outputs": [], - "source": [ - "func = lambda w: t3f.quadratic_form(A, w, w)\n", - "desired = t3f.project(t3f.matmul(t3f.transpose(A) + A, t3f.project(z, w)), w)\n", - "actual = hessian_by_vector(func, w, z)\n", - "desired, actual = sess.run([t3f.full(desired), t3f.full(actual)])\n", - "np.testing.assert_allclose(desired, actual, rtol=1e-4)" - ] + "source": [] }, { "cell_type": "code", From 3989e734788ee6a2743770b93664104c4a8735f1 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 11:11:59 +0300 Subject: [PATCH 040/233] add block diag --- t3f/Auto diff.ipynb | 167 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 136 insertions(+), 31 deletions(-) diff --git a/t3f/Auto diff.ipynb b/t3f/Auto diff.ipynb index 2404edb7..f6cca14c 100644 --- a/t3f/Auto diff.ipynb +++ b/t3f/Auto diff.ipynb @@ -34,13 +34,14 @@ "metadata": {}, "outputs": [], "source": [ - "w = t3f.random_matrix(([10] * 3, None))\n", + "d = 3\n", + "w = t3f.random_matrix(([10] * d, None))\n", "w = t3f.get_variable('w', initializer=w)\n", - "A = t3f.random_matrix(([10] * 3, [10] * 3))\n", + "A = t3f.random_matrix(([10] * d, [10] * d))\n", "A = t3f.get_variable('A', initializer=A)\n", - "z = t3f.random_matrix(([10] * 3, None))\n", + "z = t3f.random_matrix(([10] * d, None))\n", "z = t3f.get_variable('z', initializer=z)\n", - "x = t3f.random_matrix(([10] * 3, None), tt_rank=100)\n", + "x = t3f.random_matrix(([10] * d, None), tt_rank=100)\n", "x = t3f.get_variable('x', initializer=x)" ] }, @@ -159,7 +160,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -189,12 +190,14 @@ " curr_grad = cores_grad[i][r1:, :, :r2]\n", " if i < w.ndims() - 1:\n", " proj = (tf.eye(r1 * n) - q @ tf.transpose(q))\n", + " # TODO: multiply faster.\n", " delta = proj @ tf.reshape(curr_grad, (-1, r2))\n", " delta = tf.reshape(delta, left.tt_cores[i].shape)\n", " else:\n", " delta = curr_grad\n", " deltas.append(delta)\n", " return deltas_to_tangent_space(deltas, w, left, right)\n", + "\n", "def riemannian_grad(func, w):\n", " left = t3f.orthogonalize_tt_cores(w)\n", " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", @@ -214,12 +217,56 @@ " # TODO: durty hack.\n", " grad.projection_on = w\n", " return t3f.pairwise_flat_inner_projected(grad, vector_projected)[0, 0]\n", - " return riemannian_grad(new_f, w)" + " return riemannian_grad(new_f, w)\n", + "\n", + "def block_diag_hessian_by_vector(f, w, vector):\n", + " left = t3f.orthogonalize_tt_cores(w)\n", + " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", + " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", + " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", + " grad = _riemannian_grad(f, w, w_projection, left, right)\n", + " vector_projected = t3f.project(vector, w)\n", + " vector_deltas = tangent_space_to_deltas(vector_projected)\n", + " grad_deltas = tangent_space_to_deltas(grad)\n", + " final_deltas = []\n", + " for i in range(w.ndims()):\n", + " h = tf.reduce_sum(grad_deltas[i] * vector_deltas[i])\n", + " cores_grad = tf.gradients(h, w_projection.tt_cores[i])[0]\n", + " if w.is_tt_matrix():\n", + " r1, n, m, r2 = left.tt_cores[i].shape.as_list()\n", + " else:\n", + " r1, n, r2 = left.tt_cores[i].shape.as_list()\n", + " q = tf.reshape(left.tt_cores[i], (-1, r2))\n", + " if w.is_tt_matrix():\n", + " if i == 0:\n", + " curr_grad = cores_grad[:, :, :, :r2]\n", + " elif i == w.ndims() - 1:\n", + " curr_grad = cores_grad[r1:, :, :, :]\n", + " else:\n", + " curr_grad = cores_grad[r1:, :, :, :r2]\n", + " else:\n", + " if i == 0:\n", + " curr_grad = cores_grad[:, :, :r2]\n", + " elif i == w.ndims() - 1:\n", + " curr_grad = cores_grad[r1:, :, :]\n", + " else:\n", + " curr_grad = cores_grad[r1:, :, :r2]\n", + " if i < w.ndims() - 1:\n", + " proj = (tf.eye(r1 * n) - q @ tf.transpose(q))\n", + " # TODO: multiply faster.\n", + " delta = proj @ tf.reshape(curr_grad, (-1, r2))\n", + " delta = tf.reshape(delta, left.tt_cores[i].shape)\n", + " else:\n", + " delta = curr_grad\n", + " final_deltas.append(delta)\n", + " return deltas_to_tangent_space(final_deltas, w, left, right)\n", + " \n", + " " ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -229,7 +276,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -246,7 +293,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -256,45 +303,86 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "115 µs ± 7.24 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" - ] - } - ], + "outputs": [], "source": [ "%timeit sess.run(op1)" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%timeit sess.run(op2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "t1 = tf.zeros((3, 4))\n", + "t2 = tf.ones((3, 4))\n", + "h1 = tf.reduce_sum(t1 * t2)\n", + "h2 = tf.reduce_prod(t1 * t2)\n", + "tf.gradients([h1, h2], [t1, t2])" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# Try block diag hessian\n", + "deltas = tangent_space_to_deltas(t3f.project(z, w))\n", + "deltas[1:] = [tf.zeros_like(d) for d in deltas[1:]]\n", + "left = t3f.orthogonalize_tt_cores(w)\n", + "right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", + "new_z = deltas_to_tangent_space(deltas, w, left, right)\n", + "block_diag = hessian_by_vector(func, w, new_z)\n", + "actual = hessian_by_vector(func, w, z)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "op1 = hessian_by_vector(func, w, z).op\n", + "op2 = block_diag_hessian_by_vector(func, w, z).op" + ] + }, + { + "cell_type": "code", + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "106 µs ± 2.58 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" + "225 µs ± 59.9 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", + "258 µs ± 28.6 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", + "194 µs ± 23 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n", + "229 µs ± 44.5 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n" ] } ], "source": [ + "\n", + "%timeit sess.run(op1)\n", + "%timeit sess.run(op2)\n", + "\n", + "%timeit sess.run(op1)\n", "%timeit sess.run(op2)" ] }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": null, @@ -304,10 +392,27 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 14, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\n", + "1\n", + "2\n" + ] + } + ], + "source": [ + "for i in range(w.ndims()):\n", + " deltas = [tf.zeros_like(d) for d in tangent_space_to_deltas(t3f.project(z, w))]\n", + " deltas[i] = tangent_space_to_deltas(t3f.project(z, w))[i]\n", + " new_z = deltas_to_tangent_space(deltas, w, left, right)\n", + " desired = tangent_space_to_deltas(hessian_by_vector(func, w, new_z))[i]\n", + " np.testing.assert_allclose(*sess.run([desired, tangent_space_to_deltas(block_diag_hessian_by_vector(func, w, z))[i]]), rtol=1e-4)" + ] }, { "cell_type": "code", From 3736c44835407ffa4a549fee2035d594c00de47a Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 20:10:45 +0300 Subject: [PATCH 041/233] Init gradients test --- t3f/autodiff_test.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 t3f/autodiff_test.py diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py new file mode 100644 index 00000000..e7ba5c36 --- /dev/null +++ b/t3f/autodiff_test.py @@ -0,0 +1,29 @@ +import numpy as np +import tensorflow as tf + +from t3f import ops +from t3f import initializers +from t3f import variables +from t3f import riemannian +from t3f import autodiff + + +class AutodiffTest(tf.test.TestCase): + + def testGradients(self): + w = initializers.random_matrix(([5] * 3, None)) + x = initializers.random_matrix(([5] * 3, None)) + + def func(x): + return 0.5 * ops.flat_inner(x, w) ** 2 + + actual = ops.full(autodiff.gradients(func, x)) + desired = ops.full(ops.flat_inner(x, w) * riemannian.project(w, x)) + with self.test_session() as sess: + actual_v, desired_v = sess.run([actual, desired]) + np.testing.assert_allclose(actual_v, desired_v, rtol=1e-4) + + +if __name__ == "__main__": + tf.test.main() + From 1dbb0e22a964632e2e325c0f8904518d762104bf Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 20:11:08 +0300 Subject: [PATCH 042/233] Init autodiff implementation --- t3f/autodiff.py | 293 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 t3f/autodiff.py diff --git a/t3f/autodiff.py b/t3f/autodiff.py new file mode 100644 index 00000000..7d005324 --- /dev/null +++ b/t3f/autodiff.py @@ -0,0 +1,293 @@ +import tensorflow as tf + +from tensor_train import TensorTrain +import shapes +import batch_ops +import decompositions +import riemannian + + +def tangent_space_to_deltas(tt, name='t3f_tangent_space_to_deltas'): + """Convert an element of the tangent space to deltas representation. + + Tangent space elements (outputs of t3f.project) look like: + dP1 V2 ... Vd + U1 dP2 V3 ... Vd + ... + U1 ... Ud-1 dPd. + + This function takes as input an element of the tangent space and converts + it to the list of deltas [dP1, ..., dPd]. + + Args: + tt: `TensorTrain` or `TensorTrainBatch` that is a result of t3f.project, + t3f.project_matmul, or other similar functions. + name: string, name of the Op. + + Returns: + A list of delta-cores. + """ + if not hasattr(tt, 'projection_on') or tt.projection_on is None: + raise ValueError('tt argument is supposed to be a projection, but it ' + 'lacks projection_on field') + num_dims = tt.ndims() + left_tt_rank_dim = tt.left_tt_rank_dim + right_tt_rank_dim = tt.right_tt_rank_dim + deltas = [None] * num_dims + tt_ranks = shapes.lazy_tt_ranks(tt) + for i in range(1, num_dims - 1): + if int(tt_ranks[i] / 2) != tt_ranks[i] / 2: + raise ValueError('tt argument is supposed to be a projection, but its ' + 'ranks are not even.') + with tf.name_scope(name, values=tt.tt_cores): + for i in range(1, num_dims - 1): + r1, r2 = tt_ranks[i], tt_ranks[i + 1] + curr_core = tt.tt_cores[i] + slc = [slice(None)] * len(curr_core.shape) + slc[left_tt_rank_dim] = slice(int(r1 / 2), None) + slc[right_tt_rank_dim] = slice(0, int(r2 / 2)) + deltas[i] = curr_core[slice] + slc = [slice(None)] * len(tt.tt_cores[0].shape) + slc[right_tt_rank_dim] = slice(0, int(tt_ranks[1] / 2)) + deltas[0] = tt.tt_cores[0][slc] + slc = [slice(None)] * len(tt.tt_cores[0].shape) + slc[left_tt_rank_dim] = slice(int(tt_ranks[-1] / 2), None) + deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][slc] + return deltas + + +def deltas_to_tangent_space(deltas, tt, left=None, right=None, + name='t3f_deltas_to_tangent_space'): + """Converts deltas representation of tangent space vector to TensorTrain object. + + Takes as input a list of [dP1, ..., dPd] and returns + dP1 V2 ... Vd + U1 dP2 V3 ... Vd + ... + U1 ... Ud-1 dPd. + + This function is hard to use correctly because deltas should abey the + so called gauge conditions. If the don't, the function will silently return + incorrect result. This is why this function is not imported in the __init__. + + Args: + deltas: a list of deltas (essentially TT-cores). + tt: `TensorTrain` object on which the tangent space tensor represented by + delta is projected. + left: t3f.orthogonilize_tt_cores(tt). If you have it already compute, you + may pass it as argument to avoid recomputing. + right: t3f.orthogonilize_tt_cores(left, left_to_right=False). If you have + it already compute, you may pass it as argument to avoid recomputing. + name: string, name of the Op. + + Returns: + `TensorTrain` object constructed from deltas, that is from the tangent + space at point `tt`. + """ + cores = [] + dtype = deltas[0].dtype + num_dims = left.ndims() + # TODO: add cache instead of mannually pasisng precomputed stuff? + if left is None: + left = decompositions.orthogonalize_tt_cores(tt) + if right is None: + right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) + left_tangent_tt_ranks = shapes.lazy_tt_ranks(left) + right_tangent_tt_ranks = shapes.lazy_tt_ranks(left) + raw_shape = shapes.lazy_raw_shape(left) + right_rank_dim = left.right_tt_rank_dim + left_rank_dim = left.left_tt_rank_dim + is_batch_case = hasattr(tt, 'batch_size') + for i in range(num_dims): + left_tt_core = left.tt_cores[i] + right_tt_core = right.tt_cores[i] + + if i == 0: + tangent_core = tf.concat((deltas[i], left_tt_core), + axis=right_rank_dim) + elif i == num_dims - 1: + tangent_core = tf.concat((right_tt_core, deltas[i]), + axis=left_rank_dim) + else: + rank_1 = right_tangent_tt_ranks[i] + rank_2 = left_tangent_tt_ranks[i + 1] + if tt.is_tt_matrix(): + mode_size_n = raw_shape[0][i] + mode_size_m = raw_shape[1][i] + shape = [rank_1, mode_size_n, mode_size_m, rank_2] + else: + mode_size_n = raw_shape[0][i] + shape = [rank_1, mode_size_n, rank_2] + if is_batch_case: + shape = [tt.batch_size] + shape + zeros = tf.zeros(shape, dtype=dtype) + upper = tf.concat((right_tt_core, zeros), axis=right_rank_dim) + lower = tf.concat((deltas[i], left_tt_core), axis=right_rank_dim) + tangent_core = tf.concat((upper, lower), axis=left_rank_dim) + cores.append(tangent_core) + # Create instance of the same class as tt. + tangent = tt.__class__(cores) + tangent.projection_on = tt + return tangent + + +def _gradients(func, x, x_projection, left, right): + """Internal version of t3f.gradients that assumes some precomputed inputs.""" + h = func(x_projection) + cores_grad = tf.gradients(h, x_projection.tt_cores) + deltas = [] + for i in range(x.ndims()): + if x.is_tt_matrix(): + r1, n, m, r2 = left.tt_cores[i].shape.as_list() + else: + r1, n, r2 = left.tt_cores[i].shape.as_list() + q = tf.reshape(left.tt_cores[i], (-1, r2)) + if x.is_tt_matrix(): + if i == 0: + curr_grad = cores_grad[i][:, :, :, :r2] + elif i == x.ndims() - 1: + curr_grad = cores_grad[i][r1:, :, :, :] + else: + curr_grad = cores_grad[i][r1:, :, :, :r2] + else: + if i == 0: + curr_grad = cores_grad[i][:, :, :r2] + elif i == w.ndims() - 1: + curr_grad = cores_grad[i][r1:, :, :] + else: + curr_grad = cores_grad[i][r1:, :, :r2] + if i < x.ndims() - 1: + proj = (tf.eye(r1 * n) - q @ tf.transpose(q)) + # TODO: multiply faster. + delta = proj @ tf.reshape(curr_grad, (-1, r2)) + delta = tf.reshape(delta, left.tt_cores[i].shape) + else: + delta = curr_grad + deltas.append(delta) + return deltas_to_tangent_space(deltas, x, left, right) + + +def gradients(func, x, name='t3f_gradients'): + """Riemannian autodiff: returns gradient projected on tangent space of TT. + + Automatically computes projection of the gradient df/dx onto the + tangent space of TT tensor at point x. + + Warning: this is experimental feature and it may not work for some function, + e.g. ones that include QR or SVD decomposition (t3f.project, t3f.round) or + for functions that work with TT-cores directly (in contrast to working with + TT-object only via t3f functions). In this cases this function can silently + return wrong results! + + Example: + # Scalar product with some predefined tensor squared 0.5 * **2. + # It's gradient is t and it's Riemannian gradient is + # t3f.project( * t, x) + f = lambda x: 0.5 * t3f.flat_inner(x, t)**2 + projected_grad = t3f.gradients(f, x) # t3f.project(t3f.flat_inner(x, t) * t, x) + + Args: + func: function that takes TensorTrain object as input and outputs a number. + x: point at which to compute the gradient and on which tangent space to + project the gradient. + name: string, name of the Op. + + Returns: + `TensorTrain`, projection of the gradient df/dx onto the tangent space at + point x. + """ + with tf.name_scope(name, values=x.tt_cores): + left = decompositions.orthogonalize_tt_cores(x) + right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) + deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]] + x_projection = deltas_to_tangent_space(deltas, x, left, right) + return _gradients(func, x, x_projection, left, right) + + +def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product'): + """P_x d^2f/dx^2 P_x vector, i.e. Riemannian hessian by vector product. + + Automatically computes + P_x d^2f/dx^2 P_x vector + where P_x is projection onto the tangent space of TT at point x and + d^2f/dx^2 is the Hessian of the function. + + Warning: this is experimental feature and it may not work for some function, + e.g. ones that include QR or SVD decomposition (t3f.project, t3f.round) or + for functions that work with TT-cores directly (in contrast to working with + TT-object only via t3f functions). In this cases this function can silently + return wrong results! + + Example: + # Quadratic form with matrix A: . + # It's gradient is (A + A.T) x, it's Hessian is (A + A.T) + # It's Riemannian Hessian by vector product is + # t3f.project(t3f.matmul(A + t3f.transpose(A), vector), x) + f = lambda x: t3f.quadratic_form(A, x, x) + res = t3f.hessian_vector_product(f, x, vector) + + Args: + func: function that takes TensorTrain object as input and outputs a number. + x: point at which to compute the Hessian and on which tangent space to + project the gradient. + vector: `TensorTrain` object which to multiply be the Hessian. + name: string, name of the Op. + + Returns: + `TensorTrain`, projection of the gradient df/dx onto the tangent space at + point x. + """ + all_cores = list(x.tt_cores) + list(vector.tt_cores) + with tf.name_scope(name, values=all_cores): + left = decompositions.orthogonalize_tt_cores(x) + right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) + vector_projected = riemannian.project(vector, x) + vector_projected = riemannain.expand_batch_dim(vector_projected) + vector_projected.projection_on = x + + def new_f(new_x): + grad = _riemannian_grad(func, x, new_x, left, right) + grad = batch_ops.expand_batch_dim(grad) + # TODO: durty hack. + grad.projection_on = x + return riemannian.pairwise_flat_inner_projected(grad, vector_projected)[0, 0] + + return riemannian_grad(new_f, x) + + +def _block_diag_hessian_vector_product(func, x, vector): + # TODO: + left = decompositions.orthogonalize_tt_cores(x) + right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) + deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]] + x_projection = deltas_to_tangent_space(deltas, x, left, right) + grad = _gradients(func, x, x_projection, left, right) + vector_projected = riemannian.project(vector, x) + vector_deltas = tangent_space_to_deltas(vector_projected) + grad_deltas = tangent_space_to_deltas(grad) + final_deltas = [] + for i in range(x.ndims()): + h = tf.reduce_sum(grad_deltas[i] * vector_deltas[i]) + cores_grad = tf.gradients(h, x_projection.tt_cores[i])[0] + if x.is_tt_matrix(): + r1, n, m, r2 = left.tt_cores[i].shape.as_list() + else: + r1, n, r2 = left.tt_cores[i].shape.as_list() + q = tf.reshape(left.tt_cores[i], (-1, r2)) + if x.is_tt_matrix(): + if i == 0: + curr_grad = cores_grad[:, :, :, :r2] + elif i == w.ndims() - 1: + curr_grad = cores_grad[r1:, :, :, :] + else: + curr_grad = cores_grad[r1:, :, :, :r2] + else: + if i == 0: + curr_grad = cores_grad[:, :, :r2] + elif i == x.ndims() - 1: + curr_grad = cores_grad[r1:, :, :] + else: + curr_grad = cores_grad[r1:, :, :r2] + if i < x.ndims() - 1: + proj = (tf.eye(r1 * n) - q @ tf.transpose(q)) + # TODO: multiply faster. + delta = tf.matmul(proj, tf.reshape(curr_grad, (-1, r2))) + delta = tf.reshape(delta, left.tt_cores[i].shape) + else: + delta = curr_grad + final_deltas.append(delta) + return deltas_to_tangent_space(final_deltas, x, left, right) From af6840ede2333501a27a13763a2801024ca051c2 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 20:20:41 +0300 Subject: [PATCH 043/233] Add quadratic form test --- t3f/autodiff_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index e7ba5c36..b2f9706c 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -12,7 +12,9 @@ class AutodiffTest(tf.test.TestCase): def testGradients(self): w = initializers.random_matrix(([5] * 3, None)) + A = initializers.random_matrix(([5] * 3, [5] * 3)) x = initializers.random_matrix(([5] * 3, None)) + z = initializers.random_matrix(([5] * 3, None)) def func(x): return 0.5 * ops.flat_inner(x, w) ** 2 @@ -23,6 +25,17 @@ def func(x): actual_v, desired_v = sess.run([actual, desired]) np.testing.assert_allclose(actual_v, desired_v, rtol=1e-4) + def func2(x): + return ops.quadratic_form(A, x, x) + + actual2 = ops.full(autodiff.gradients(func2, x)) + projected_vector = riemannian.project(z, x) + hessian_by_vector = ops.matmul(ops.transpose(A) + A, projected_vector) + desired2 = ops.full(riemannian.project(hessian_by_vector, x)) + with self.test_session() as sess: + actual_v2, desired_v2 = sess.run([actual2, desired2]) + np.testing.assert_allclose(actual_v2, desired_v2, rtol=1e-4) + if __name__ == "__main__": tf.test.main() From 2e0cbd5240e07e1a62b56ba1be9cc85e2520f171 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 20:21:57 +0300 Subject: [PATCH 044/233] Add autodiff to init --- t3f/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/t3f/__init__.py b/t3f/__init__.py index 60abb533..bc07724d 100644 --- a/t3f/__init__.py +++ b/t3f/__init__.py @@ -69,13 +69,18 @@ from t3f.decompositions import to_tt_matrix from t3f.decompositions import to_tt_tensor +from t3f.autodiff import tangent_space_to_deltas +from t3f.autodiff import gradients +from t3f.autodiff import hessian_vector_product + import t3f.approximate as approximate import t3f.kronecker as kronecker import t3f.utils as utils _directly_imported = ['tensor_train_base', 'tensor_train', 'tensor_train_batch', 'variables', 'ops', 'batch_ops', 'initializers', - 'regularizers', 'riemannian', 'shapes', 'decompositions'] + 'regularizers', 'riemannian', 'shapes', 'decompositions', + 'autodiff'] __all__ = [s for s in dir() if s not in _directly_imported and not s.startswith('_')] From a943a28260f1642cce01e822c578b815c9cd2b3a Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 20:22:37 +0300 Subject: [PATCH 045/233] Add todo about moving tangent_space_to_deltas --- t3f/autodiff.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 7d005324..5954877d 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -7,6 +7,7 @@ import riemannian +# TODO: move to riemannian. def tangent_space_to_deltas(tt, name='t3f_tangent_space_to_deltas'): """Convert an element of the tangent space to deltas representation. @@ -53,6 +54,7 @@ def tangent_space_to_deltas(tt, name='t3f_tangent_space_to_deltas'): return deltas +# TODO: move to riemannian. def deltas_to_tangent_space(deltas, tt, left=None, right=None, name='t3f_deltas_to_tangent_space'): """Converts deltas representation of tangent space vector to TensorTrain object. From 6fb1d8412df9479f3a33831828bf020ce3ce1ebe Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 20:59:11 +0300 Subject: [PATCH 046/233] Fix second gradient test --- t3f/autodiff_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index b2f9706c..ea06ac22 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -29,9 +29,8 @@ def func2(x): return ops.quadratic_form(A, x, x) actual2 = ops.full(autodiff.gradients(func2, x)) - projected_vector = riemannian.project(z, x) - hessian_by_vector = ops.matmul(ops.transpose(A) + A, projected_vector) - desired2 = ops.full(riemannian.project(hessian_by_vector, x)) + grad = ops.matmul(ops.transpose(A) + A, x) + desired2 = ops.full(riemannian.project(grad, x)) with self.test_session() as sess: actual_v2, desired_v2 = sess.run([actual2, desired2]) np.testing.assert_allclose(actual_v2, desired_v2, rtol=1e-4) From 3fff8b8f853efec24b71cf8d468b648541966bb2 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 21:09:29 +0300 Subject: [PATCH 047/233] Add hessian by vector test --- t3f/autodiff_test.py | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index ea06ac22..e67939ed 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -16,14 +16,14 @@ def testGradients(self): x = initializers.random_matrix(([5] * 3, None)) z = initializers.random_matrix(([5] * 3, None)) - def func(x): + def func1(x): return 0.5 * ops.flat_inner(x, w) ** 2 - actual = ops.full(autodiff.gradients(func, x)) - desired = ops.full(ops.flat_inner(x, w) * riemannian.project(w, x)) + actual1 = ops.full(autodiff.gradients(func1, x)) + desired1 = ops.full(ops.flat_inner(x, w) * riemannian.project(w, x)) with self.test_session() as sess: - actual_v, desired_v = sess.run([actual, desired]) - np.testing.assert_allclose(actual_v, desired_v, rtol=1e-4) + actual1_v, desired1_v = sess.run([actual1, desired1]) + np.testing.assert_allclose(actual1_v, desired1_v, rtol=1e-4) def func2(x): return ops.quadratic_form(A, x, x) @@ -35,6 +35,37 @@ def func2(x): actual_v2, desired_v2 = sess.run([actual2, desired2]) np.testing.assert_allclose(actual_v2, desired_v2, rtol=1e-4) + def testHessianVectorProduct(self): + w = initializers.random_matrix(([5] * 3, None)) + A = initializers.random_matrix(([5] * 3, [5] * 3)) + x = initializers.random_matrix(([5] * 3, None)) + z = initializers.random_matrix(([5] * 3, None)) + projected_vector = ops.full(riemannian.project(z, x)) + + def func1(x): + return 0.5 * ops.flat_inner(x, w) ** 2 + + actual1 = ops.full(autodiff.hessian_vector_product(func1, x, z)) + # Grad: w + # Hessian: w w.T + w_full = ops.full(w) + hessian = ops.flat_inner(w, x) * tf.matmul(w_full, tf.transpose(w_full)) + desired1 = tf.matmul(hessian, projected_vector) + with self.test_session() as sess: + actual1_v, desired1_v = sess.run([actual1, desired1]) + np.testing.assert_allclose(actual1_v, desired1_v, rtol=1e-4) + + def func2(x): + return ops.quadratic_form(A, x, x) + + actual2 = ops.full(autodiff.hessian_vector_product(func2, x, z)) + projected_vector = riemannian.project(z, x) + hessian_by_vector = ops.matmul(ops.transpose(A) + A, projected_vector) + desired2 = ops.full(riemannian.project(hessian_by_vector, x)) + with self.test_session() as sess: + actual2_v, desired2_v = sess.run([actual2, desired2]) + np.testing.assert_allclose(actual2_v, desired2_v, rtol=1e-3) + if __name__ == "__main__": tf.test.main() From 94be468701a6903c4775291da9d30076ded97e33 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 21:09:43 +0300 Subject: [PATCH 048/233] Fix typos in hessian_by_vector --- t3f/autodiff.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 5954877d..e64291fa 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -238,17 +238,17 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product'): left = decompositions.orthogonalize_tt_cores(x) right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) vector_projected = riemannian.project(vector, x) - vector_projected = riemannain.expand_batch_dim(vector_projected) + vector_projected = shapes.expand_batch_dim(vector_projected) vector_projected.projection_on = x def new_f(new_x): - grad = _riemannian_grad(func, x, new_x, left, right) - grad = batch_ops.expand_batch_dim(grad) + grad = _gradients(func, x, new_x, left, right) + grad = shapes.expand_batch_dim(grad) # TODO: durty hack. grad.projection_on = x return riemannian.pairwise_flat_inner_projected(grad, vector_projected)[0, 0] - return riemannian_grad(new_f, x) + return gradients(new_f, x) def _block_diag_hessian_vector_product(func, x, vector): From 41429a507b076c4f155809105f1a03a28b6b4289 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 21:28:45 +0300 Subject: [PATCH 049/233] Add see also --- t3f/autodiff.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index e64291fa..e6df08f6 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -191,6 +191,9 @@ def gradients(func, x, name='t3f_gradients'): Returns: `TensorTrain`, projection of the gradient df/dx onto the tangent space at point x. + + See also: + t3f.hessian_vector_product """ with tf.name_scope(name, values=x.tt_cores): left = decompositions.orthogonalize_tt_cores(x) @@ -232,6 +235,9 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product'): Returns: `TensorTrain`, projection of the gradient df/dx onto the tangent space at point x. + + See also: + t3f.gradients """ all_cores = list(x.tt_cores) + list(vector.tt_cores) with tf.name_scope(name, values=all_cores): From 5fdf2308a5fc98856716e058881a981d573b1056 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Mon, 14 May 2018 21:29:07 +0300 Subject: [PATCH 050/233] Add manifold curvature disclaimer --- t3f/autodiff.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index e6df08f6..a5d56cae 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -211,6 +211,9 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product'): where P_x is projection onto the tangent space of TT at point x and d^2f/dx^2 is the Hessian of the function. + Note that the true hessian also includes the manifold curvature term + which is ignored here. + Warning: this is experimental feature and it may not work for some function, e.g. ones that include QR or SVD decomposition (t3f.project, t3f.round) or for functions that work with TT-cores directly (in contrast to working with From 24db83fe8e3d23a6a75e8d1739023a8e69736ff8 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Tue, 15 May 2018 11:04:16 +0300 Subject: [PATCH 051/233] Fix imports --- t3f/autodiff.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index a5d56cae..16f04b1a 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -1,10 +1,10 @@ import tensorflow as tf -from tensor_train import TensorTrain -import shapes -import batch_ops -import decompositions -import riemannian +from t3f.tensor_train import TensorTrain +from t3f import shapes +from t3f import batch_ops +from t3f import decompositions +from t3f import riemannian # TODO: move to riemannian. From e2c8ecfa45df6e736d6ea09ab299bc19c8d1448b Mon Sep 17 00:00:00 2001 From: bihaqo Date: Tue, 15 May 2018 11:33:55 +0300 Subject: [PATCH 052/233] Multiply faster --- t3f/autodiff.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 16f04b1a..46316f6f 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -153,9 +153,9 @@ def _gradients(func, x, x_projection, left, right): else: curr_grad = cores_grad[i][r1:, :, :r2] if i < x.ndims() - 1: - proj = (tf.eye(r1 * n) - q @ tf.transpose(q)) - # TODO: multiply faster. - delta = proj @ tf.reshape(curr_grad, (-1, r2)) + delta = curr_grad + delta = tf.reshape(delta, (-1, r2)) + delta -= tf.matmul(q, tf.matmul(tf.transpose(q), delta)) delta = tf.reshape(delta, left.tt_cores[i].shape) else: delta = curr_grad From 5515066bc920a11c1f371cb96ae8006e90c762f2 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Thu, 17 May 2018 16:08:16 +0300 Subject: [PATCH 053/233] bug fix in hessian by vector test (still does not pass) --- t3f/autodiff_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index e67939ed..0ee371cb 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -44,20 +44,20 @@ def testHessianVectorProduct(self): def func1(x): return 0.5 * ops.flat_inner(x, w) ** 2 + # Grad: w + # Hessian: w w.T + # Hessian by vector: w actual1 = ops.full(autodiff.hessian_vector_product(func1, x, z)) - # Grad: w - # Hessian: w w.T - w_full = ops.full(w) - hessian = ops.flat_inner(w, x) * tf.matmul(w_full, tf.transpose(w_full)) - desired1 = tf.matmul(hessian, projected_vector) + desired1 = riemannian.project(ops.flat_inner(z, w) * w, x) + desired1 = ops.full(desired1) with self.test_session() as sess: actual1_v, desired1_v = sess.run([actual1, desired1]) np.testing.assert_allclose(actual1_v, desired1_v, rtol=1e-4) def func2(x): return ops.quadratic_form(A, x, x) - + # Hessian of is A + A.T actual2 = ops.full(autodiff.hessian_vector_product(func2, x, z)) projected_vector = riemannian.project(z, x) hessian_by_vector = ops.matmul(ops.transpose(A) + A, projected_vector) From a241a183d330e44715771e8a0ef8f4d7dc4e2264 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Thu, 17 May 2018 16:09:01 +0300 Subject: [PATCH 054/233] try to do block diagonal faster --- t3f/Auto diff.ipynb | 191 ++++++++++++++++++++++++++++++-------------- 1 file changed, 130 insertions(+), 61 deletions(-) diff --git a/t3f/Auto diff.ipynb b/t3f/Auto diff.ipynb index f6cca14c..c39acd17 100644 --- a/t3f/Auto diff.ipynb +++ b/t3f/Auto diff.ipynb @@ -67,7 +67,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -160,7 +160,47 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([1, 2])" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.array([[1, 2], [3, 4]])[0, :]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([1, 2, 3, 4])" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.array([[1, 2], [3, 4]]).flatten()" + ] + }, + { + "cell_type": "code", + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -222,35 +262,59 @@ "def block_diag_hessian_by_vector(f, w, vector):\n", " left = t3f.orthogonalize_tt_cores(w)\n", " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", - " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", + " original_zeros = []\n", + " deltas = []\n", + " for i in range(w.ndims()):\n", + " shape = right.tt_cores[i].get_shape().as_list()\n", + " shape += [w.ndims()]\n", + " curr_original_zero = tf.zeros(shape)\n", + " original_zeros.append(curr_original_zero)\n", + " delta = tf.reduce_sum(curr_original_zero, axis=-1)\n", + " deltas.append(delta)\n", + " deltas[0] += right.tt_cores[0]\n", " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", - " grad = _riemannian_grad(f, w, w_projection, left, right)\n", + "# grad = _riemannian_grad(f, w, w_projection, left, right)\n", " vector_projected = t3f.project(vector, w)\n", " vector_deltas = tangent_space_to_deltas(vector_projected)\n", - " grad_deltas = tangent_space_to_deltas(grad)\n", + "# grad_deltas = tangent_space_to_deltas(grad)\n", + " h = f(w_projection)\n", + " cores_grad = tf.gradients(h, original_zeros)\n", + " grad_deltas = []\n", + " for i in range(w.ndims()):\n", + " if w.is_tt_matrix():\n", + " r1, n, m, r2 = left.tt_cores[i].shape.as_list()\n", + " else:\n", + " r1, n, r2 = left.tt_cores[i].shape.as_list()\n", + " q = tf.reshape(left.tt_cores[i], (-1, r2))\n", + " if w.is_tt_matrix():\n", + " curr_grad = cores_grad[i][:, :, :, :, i]\n", + " else:\n", + " curr_grad = cores_grad[i][:, :, :, i]\n", + " if i < w.ndims() - 1:\n", + " proj = (tf.eye(r1 * n) - q @ tf.transpose(q))\n", + " # TODO: multiply faster.\n", + " delta = proj @ tf.reshape(curr_grad, (-1, r2))\n", + " delta = tf.reshape(delta, left.tt_cores[i].shape)\n", + " else:\n", + " delta = curr_grad\n", + " grad_deltas.append(delta)\n", + " \n", + " intermidiate_f = 0.0\n", + " for i in range(w.ndims()):\n", + " intermidiate_f += tf.reduce_sum(grad_deltas[i] * vector_deltas[i])\n", " final_deltas = []\n", + " cores_grad = tf.gradients(intermidiate_f, original_zeros)\n", " for i in range(w.ndims()):\n", - " h = tf.reduce_sum(grad_deltas[i] * vector_deltas[i])\n", - " cores_grad = tf.gradients(h, w_projection.tt_cores[i])[0]\n", " if w.is_tt_matrix():\n", " r1, n, m, r2 = left.tt_cores[i].shape.as_list()\n", " else:\n", " r1, n, r2 = left.tt_cores[i].shape.as_list()\n", " q = tf.reshape(left.tt_cores[i], (-1, r2))\n", " if w.is_tt_matrix():\n", - " if i == 0:\n", - " curr_grad = cores_grad[:, :, :, :r2]\n", - " elif i == w.ndims() - 1:\n", - " curr_grad = cores_grad[r1:, :, :, :]\n", - " else:\n", - " curr_grad = cores_grad[r1:, :, :, :r2]\n", + " curr_grad = cores_grad[i][:, :, :, :, i]\n", " else:\n", - " if i == 0:\n", - " curr_grad = cores_grad[:, :, :r2]\n", - " elif i == w.ndims() - 1:\n", - " curr_grad = cores_grad[r1:, :, :]\n", - " else:\n", - " curr_grad = cores_grad[r1:, :, :r2]\n", + " curr_grad = cores_grad[i][:, :, :, i]\n", + "# import pdb; pdb.set_trace()\n", " if i < w.ndims() - 1:\n", " proj = (tf.eye(r1 * n) - q @ tf.transpose(q))\n", " # TODO: multiply faster.\n", @@ -266,7 +330,52 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 46, + "metadata": {}, + "outputs": [], + "source": [ + "# Try block diag hessian\n", + "deltas = tangent_space_to_deltas(t3f.project(z, w))\n", + "deltas[1:] = [tf.zeros_like(d) for d in deltas[1:]]\n", + "left = t3f.orthogonalize_tt_cores(w)\n", + "right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", + "new_z = deltas_to_tangent_space(deltas, w, left, right)\n", + "block_diag = hessian_by_vector(func, w, new_z)\n", + "actual = hessian_by_vector(func, w, z)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [ + { + "ename": "AssertionError", + "evalue": "\nNot equal to tolerance rtol=0.0001, atol=0\n\n(mismatch 100.0%)\n x: array([[[[14.918402, -4.146306]],\n\n [[ 1.740906, -5.047251]],...\n y: array([[[[ 19.173162, -6.58578 ]],\n\n [[ -2.820656, -1.763076]],...", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mAssertionError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[0mdesired\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtangent_space_to_deltas\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhessian_by_vector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_z\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 6\u001b[0m \u001b[1;31m# print(desired, tangent_space_to_deltas(block_diag_hessian_by_vector(func, w, z)))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtesting\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0massert_allclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdesired\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtangent_space_to_deltas\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mblock_diag_hessian_by_vector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mz\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrtol\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1e-4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;32m~\\AppData\\Local\\Continuum\\miniconda3\\lib\\site-packages\\numpy\\testing\\nose_tools\\utils.py\u001b[0m in \u001b[0;36massert_allclose\u001b[1;34m(actual, desired, rtol, atol, equal_nan, err_msg, verbose)\u001b[0m\n\u001b[0;32m 1394\u001b[0m \u001b[0mheader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'Not equal to tolerance rtol=%g, atol=%g'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mrtol\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0matol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1395\u001b[0m assert_array_compare(compare, actual, desired, err_msg=str(err_msg),\n\u001b[1;32m-> 1396\u001b[1;33m verbose=verbose, header=header, equal_nan=equal_nan)\n\u001b[0m\u001b[0;32m 1397\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1398\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\AppData\\Local\\Continuum\\miniconda3\\lib\\site-packages\\numpy\\testing\\nose_tools\\utils.py\u001b[0m in \u001b[0;36massert_array_compare\u001b[1;34m(comparison, x, y, err_msg, verbose, header, precision, equal_nan, equal_inf)\u001b[0m\n\u001b[0;32m 777\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mheader\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 778\u001b[0m names=('x', 'y'), precision=precision)\n\u001b[1;32m--> 779\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mAssertionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 780\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 781\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtraceback\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mAssertionError\u001b[0m: \nNot equal to tolerance rtol=0.0001, atol=0\n\n(mismatch 100.0%)\n x: array([[[[14.918402, -4.146306]],\n\n [[ 1.740906, -5.047251]],...\n y: array([[[[ 19.173162, -6.58578 ]],\n\n [[ -2.820656, -1.763076]],..." + ] + } + ], + "source": [ + "for i in range(w.ndims()):\n", + " deltas = [tf.zeros_like(d) for d in tangent_space_to_deltas(t3f.project(z, w))]\n", + " deltas[i] = tangent_space_to_deltas(t3f.project(z, w))[i]\n", + " new_z = deltas_to_tangent_space(deltas, w, left, right)\n", + " desired = tangent_space_to_deltas(hessian_by_vector(func, w, new_z))[i]\n", + "# print(desired, tangent_space_to_deltas(block_diag_hessian_by_vector(func, w, z)))\n", + " np.testing.assert_allclose(*sess.run([desired, tangent_space_to_deltas(block_diag_hessian_by_vector(func, w, z))[i]]), rtol=1e-4)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -276,7 +385,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -332,22 +441,6 @@ "tf.gradients([h1, h2], [t1, t2])" ] }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# Try block diag hessian\n", - "deltas = tangent_space_to_deltas(t3f.project(z, w))\n", - "deltas[1:] = [tf.zeros_like(d) for d in deltas[1:]]\n", - "left = t3f.orthogonalize_tt_cores(w)\n", - "right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", - "new_z = deltas_to_tangent_space(deltas, w, left, right)\n", - "block_diag = hessian_by_vector(func, w, new_z)\n", - "actual = hessian_by_vector(func, w, z)" - ] - }, { "cell_type": "code", "execution_count": 12, @@ -390,30 +483,6 @@ "outputs": [], "source": [] }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\n", - "1\n", - "2\n" - ] - } - ], - "source": [ - "for i in range(w.ndims()):\n", - " deltas = [tf.zeros_like(d) for d in tangent_space_to_deltas(t3f.project(z, w))]\n", - " deltas[i] = tangent_space_to_deltas(t3f.project(z, w))[i]\n", - " new_z = deltas_to_tangent_space(deltas, w, left, right)\n", - " desired = tangent_space_to_deltas(hessian_by_vector(func, w, new_z))[i]\n", - " np.testing.assert_allclose(*sess.run([desired, tangent_space_to_deltas(block_diag_hessian_by_vector(func, w, z))[i]]), rtol=1e-4)" - ] - }, { "cell_type": "code", "execution_count": null, From e3cdb99dc0bff93729c754bb09bacc45e60c3fb0 Mon Sep 17 00:00:00 2001 From: bihaqo Date: Thu, 17 May 2018 16:13:26 +0300 Subject: [PATCH 055/233] Fix big in replace_tf_svd (now correcty transpose v) --- t3f/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/t3f/utils.py b/t3f/utils.py index 7f828538..e39e6194 100644 --- a/t3f/utils.py +++ b/t3f/utils.py @@ -29,9 +29,11 @@ def my_svd(tensor, full_matrices=False, compute_uv=True): s_, u_, v_ = tf.original_svd(tensor, full_matrices, compute_uv) s = tf.reshape(s, s_.get_shape()) u = tf.reshape(u, u_.get_shape()) - v = tf.reshape(v, v_.get_shape()) + v_shape = v_.get_shape().as_list() + v_shape[-2], v_shape[-1] = v_shape[-1], v_shape[-2] + v = tf.reshape(v, v_shape) # Converting numpy order of v dims to TF order. - order = range(tensor.get_shape().ndims) + order = list(range(tensor.get_shape().ndims)) order[-2], order[-1] = order[-1], order[-2] v = tf.transpose(v, order) return s, u, v From 0ae3bcf8041c455b38536b9a8da110dbf7c75a5f Mon Sep 17 00:00:00 2001 From: bihaqo Date: Thu, 17 May 2018 16:29:55 +0300 Subject: [PATCH 056/233] Add test for replace_tf_svd_with_np_svd --- t3f/utils_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/t3f/utils_test.py b/t3f/utils_test.py index 170ab420..5bc7dce7 100644 --- a/t3f/utils_test.py +++ b/t3f/utils_test.py @@ -21,6 +21,16 @@ def testUnravelIndex(self): actual = utils.unravel_index(linear_idx, shape) self.assertAllEqual(desired, actual.eval()) + def testReplaceTfSvdWithNpSvd(self): + with self.test_session() as sess: + mat = tf.constant([[3., 4], [5, 6]]) + desired = sess.run(tf.svd(mat)) + utils.replace_tf_svd_with_np_svd() + actual = sess.run(tf.svd(mat)) + self.assertAllClose(actual[0], desired[0]) + self.assertAllClose(np.abs(np.dot(actual[1].T, desired[1])), np.eye(2)) + self.assertAllClose(np.abs(np.dot(actual[2].T, desired[2])), np.eye(2)) + if __name__ == "__main__": tf.test.main() From 06d9faf4cbcec0124d535f2c4fb0c951bb1bb514 Mon Sep 17 00:00:00 2001 From: James Oldfield Date: Thu, 7 Jun 2018 15:10:13 +0100 Subject: [PATCH 057/233] tf.mul -> tf.multiply --- t3f/regularizers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/regularizers.py b/t3f/regularizers.py index 6928e329..26b8cf67 100644 --- a/t3f/regularizers.py +++ b/t3f/regularizers.py @@ -34,7 +34,7 @@ def l2(tt): my_scale = tf.convert_to_tensor(scale, dtype=tt.dtype.base_dtype, name='scale') - return tf.mul(my_scale, ops.frobenius_norm_squared(tt), name=name) + return tf.multiply(my_scale, ops.frobenius_norm_squared(tt), name=name) return l2 @@ -74,6 +74,6 @@ def regularizer(tt): penalty = 0.0 for i in range(tt.ndims()): penalty += core_regularizer(tt.tt_cores[i]) - return tf.mul(my_scale, penalty, name=name) + return tf.multiply(my_scale, penalty, name=name) return regularizer From e55356e4ca876685997444139fca53aeb92e93d0 Mon Sep 17 00:00:00 2001 From: Yermek Kapushev Date: Tue, 17 Jul 2018 15:36:00 +0300 Subject: [PATCH 058/233] * Fix support for float64 in gather_nd --- t3f/ops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index e57fd9ab..6d9e7fed 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -315,7 +315,7 @@ def tt_tt_flat_inner(tt_a, tt_b): """Inner product between two TT-tensors or TT-matrices along all axis. The shapes of tt_a and tt_b should coincide. - + Args: tt_a: `TensorTrain` or `TensorTrainBatch` object tt_b: `TensorTrain` or `TensorTrainBatch` object @@ -328,7 +328,7 @@ def tt_tt_flat_inner(tt_a, tt_b): ValueError if the arguments are not `TensorTrain` objects, have different number of TT-cores, different underlying shape, or if you are trying to compute inner product between a TT-matrix and a TT-tensor. - + Complexity: Multiplying two single TT-objects is O(d r^3 n) where d is the number of TT-cores (tt_a.ndims()), r is the largest TT-rank @@ -1028,7 +1028,7 @@ def quadratic_form(A, b, c): Raises: ValueError if the arguments are not TT-matrices or if the shapes are not consistent. - + Complexity: O(batch_size r_A r_c r_b n d (r_b + r_A n + r_c)) d is the number of TT-cores (A.ndims()); @@ -1148,7 +1148,7 @@ def gather_nd(tt, indices): raise ValueError('The last dimension of indices (%d) should have ' 'the same size as the number of dimensions in the tt ' 'object (%d).' % (indices.get_shape()[-1], tt.ndims())) - tt_elements = tf.ones(tf.shape(indices)[:-1]) + tt_elements = tf.ones(tf.shape(indices)[:-1], dtype=tt.dtype) tt_elements = tf.reshape(tt_elements, (-1, 1, 1)) for core_idx in range(tt.ndims()): curr_core = tt.tt_cores[core_idx] From 7459eba9b64533fbe911a1af54b09012f2f0484d Mon Sep 17 00:00:00 2001 From: Vishnu Lokhande Date: Wed, 8 Aug 2018 14:40:10 -0500 Subject: [PATCH 059/233] Line 141, add np.any() --- t3f/decompositions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/decompositions.py b/t3f/decompositions.py index b16ee7f0..1c0a70bf 100644 --- a/t3f/decompositions.py +++ b/t3f/decompositions.py @@ -138,7 +138,7 @@ def to_tt_tensor(tens, max_tt_rank=10, epsilon=None): # Raises ValueError if ndims is not defined. d = static_shape.__len__() max_tt_rank = np.array(max_tt_rank).astype(np.int32) - if max_tt_rank < 1: + if np.any(max_tt_rank < 1): raise ValueError('Maximum TT-rank should be greater or equal to 1.') if epsilon is not None and epsilon < 0: raise ValueError('Epsilon should be non-negative.') From 76aeaa7989b62263283d1749aadf4c369e7d21f5 Mon Sep 17 00:00:00 2001 From: Vishnu Lokhande Date: Tue, 14 Aug 2018 11:38:01 -0500 Subject: [PATCH 060/233] add tests to handle composite ranks --- t3f/decompositions_test.py | 40 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/t3f/decompositions_test.py b/t3f/decompositions_test.py index 158eff9e..0ba17e4e 100644 --- a/t3f/decompositions_test.py +++ b/t3f/decompositions_test.py @@ -62,6 +62,43 @@ def testTTVector(self): with self.test_session(): self.assertAllClose(vec, ops.full(tt_vec).eval()) + + def testTTCompositeRank(self): + # Test if a composite rank (list of ranks) can be used for decomposition + np_tensor = np.random.rand(2,3,3,1) + tf_tensor = tf.constant(np_tensor) + + tt_ranks = [1,2,3,3,1] + tt_tensor = decompositions.to_tt_tensor(tf_tensor, max_tt_rank = tt_ranks) + with self.test_session(): + self.assertAllClose(np_tensor, ops.full(tt_tensor).eval()) + + def testTTCompositeRankTensor(self): + # Test if a composite rank (list of ranks) can be used for decomposition for tensor + np.random.seed(1) + np_tensor = np.random.rand(2,3,3,1) + tf_tensor = tf.constant(np_tensor) + + tt_ranks = [1,2,3,3,1] + tt_tensor = decompositions.to_tt_tensor(tf_tensor, max_tt_rank = tt_ranks) + with self.test_session(): + self.assertAllClose(np_tensor, ops.full(tt_tensor).eval()) + + def testTTCompositeRankMatrix(self): + # Test if a composite rank (list of ranks) can be used for decomposition for a matrix + inp_shape = (2, 3, 3, 2) + out_shape = (1, 2, 2, 1) + np.random.seed(1) + mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) + mat = mat.astype(np.float32) + tf_mat = tf.constant(mat) + tt_ranks = [10,20,30,40,30] + tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), + max_tt_rank=tt_ranks) + with self.test_session(): + self.assertAllClose(mat, ops.full(tt_mat).eval(), atol=1e-5, rtol=1e-5) + + def testTTMatrix(self): # Convert a np.prod(out_shape) x np.prod(in_shape) matrix into TT-matrix # and back. @@ -71,8 +108,7 @@ def testTTMatrix(self): mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) mat = mat.astype(np.float32) tf_mat = tf.constant(mat) - tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), - max_tt_rank=90) + tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), max_tt_rank=90) with self.test_session(): # TODO: why so bad accuracy? self.assertAllClose(mat, ops.full(tt_mat).eval(), atol=1e-5, rtol=1e-5) From ec60f0345fd629049175523a36b30a7dbf790618 Mon Sep 17 00:00:00 2001 From: Vishnu Lokhande Date: Sun, 26 Aug 2018 22:49:38 -0500 Subject: [PATCH 061/233] Add tests for composite ranks as per style --- t3f/decompositions_test.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/t3f/decompositions_test.py b/t3f/decompositions_test.py index 0ba17e4e..5848cdaf 100644 --- a/t3f/decompositions_test.py +++ b/t3f/decompositions_test.py @@ -62,42 +62,32 @@ def testTTVector(self): with self.test_session(): self.assertAllClose(vec, ops.full(tt_vec).eval()) - - def testTTCompositeRank(self): - # Test if a composite rank (list of ranks) can be used for decomposition - np_tensor = np.random.rand(2,3,3,1) - tf_tensor = tf.constant(np_tensor) - - tt_ranks = [1,2,3,3,1] - tt_tensor = decompositions.to_tt_tensor(tf_tensor, max_tt_rank = tt_ranks) - with self.test_session(): - self.assertAllClose(np_tensor, ops.full(tt_tensor).eval()) - def testTTCompositeRankTensor(self): - # Test if a composite rank (list of ranks) can be used for decomposition for tensor + # Test if a composite rank (list of ranks) can be used for decomposition + # for tensor. np.random.seed(1) - np_tensor = np.random.rand(2,3,3,1) + np_tensor = np.random.rand(2, 3, 3, 1) tf_tensor = tf.constant(np_tensor) - tt_ranks = [1,2,3,3,1] - tt_tensor = decompositions.to_tt_tensor(tf_tensor, max_tt_rank = tt_ranks) + tt_ranks = [1, 2, 3, 3, 1] + tt_tensor = decompositions.to_tt_tensor(tf_tensor, max_tt_rank=tt_ranks) with self.test_session(): self.assertAllClose(np_tensor, ops.full(tt_tensor).eval()) def testTTCompositeRankMatrix(self): - # Test if a composite rank (list of ranks) can be used for decomposition for a matrix + # Test if a composite rank (list of ranks) can be used for decomposition + # for a matrix. inp_shape = (2, 3, 3, 2) out_shape = (1, 2, 2, 1) np.random.seed(1) mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) mat = mat.astype(np.float32) tf_mat = tf.constant(mat) - tt_ranks = [10,20,30,40,30] + tt_ranks = [10, 20, 30, 40, 30] tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), max_tt_rank=tt_ranks) with self.test_session(): self.assertAllClose(mat, ops.full(tt_mat).eval(), atol=1e-5, rtol=1e-5) - def testTTMatrix(self): # Convert a np.prod(out_shape) x np.prod(in_shape) matrix into TT-matrix @@ -108,7 +98,8 @@ def testTTMatrix(self): mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) mat = mat.astype(np.float32) tf_mat = tf.constant(mat) - tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), max_tt_rank=90) + tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), + max_tt_rank=90) with self.test_session(): # TODO: why so bad accuracy? self.assertAllClose(mat, ops.full(tt_mat).eval(), atol=1e-5, rtol=1e-5) From f9995cba273bfe4f0de5b3b2f419d87a8af9cc25 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 21 Oct 2018 12:24:54 +0100 Subject: [PATCH 062/233] init doc structure --- docs/api.rst | 42 ++++++++++++++++++++++++++++++++++++++++++ docs/conf.py | 7 ++++++- docs/index.rst | 43 +++++++------------------------------------ docs/installation.rst | 17 +++++++++++++++++ 4 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 docs/api.rst create mode 100644 docs/installation.rst diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..19cea1ac --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,42 @@ +.. t3f documentation master file, created by + sphinx-quickstart on Sun Mar 12 10:06:09 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +API +=== + +Here you can find description of functions and methods avaliable in t3f. + +Module contents +--------------- + +.. automodule:: t3f + :members: + :undoc-members: + :show-inheritance: + + +t3f\.utils module +----------------- + +.. automodule:: t3f.utils + :members: + :undoc-members: + :show-inheritance: + +t3f\.kronecker module +--------------------- + +.. automodule:: t3f.kronecker + :members: + :undoc-members: + :show-inheritance: + +t3f\.approximate module +----------------------- + +.. automodule:: t3f.approximate + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py index b450933b..fc9d3031 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,6 +26,11 @@ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' +is_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if is_rtd: + html_theme = 'default' +else: + html_theme = 'sphinx_rtd_theme' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -86,7 +91,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +# html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/index.rst b/docs/index.rst index 8ec97263..92d4a1f8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,40 +1,11 @@ -.. t3f documentation master file, created by - sphinx-quickstart on Sun Mar 12 10:06:09 2017. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to t3f documentation! +t3f - library for working with tensor train decomposition built on top of TensorFlow ============================= -Module contents ---------------- - -.. automodule:: t3f - :members: - :undoc-members: - :show-inheritance: - - -t3f\.utils module ------------------ - -.. automodule:: t3f.utils - :members: - :undoc-members: - :show-inheritance: - -t3f\.kronecker module ---------------------- - -.. automodule:: t3f.kronecker - :members: - :undoc-members: - :show-inheritance: +Hi! -t3f\.approximate module ------------------------ +.. toctree:: + :maxdepth: 1 + :caption: Getting started -.. automodule:: t3f.approximate - :members: - :undoc-members: - :show-inheritance: + installation + api \ No newline at end of file diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 00000000..25982e6d --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,17 @@ +.. t3f documentation master file, created by + sphinx-quickstart on Sun Mar 12 10:06:09 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Installation +============ + +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, supported versions are from 1.0 to 1.7 (see [here](https://www.tensorflow.org/versions/r1.7/install/) for TF 1.7 installation instructions). + +We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. + +Then simply run + +```bash +pip install t3f +``` \ No newline at end of file From f638a2d0f4e9c2ced2f837e2be16d0e1e57fae94 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 27 Oct 2018 19:47:55 +0100 Subject: [PATCH 063/233] use rst syntaxis --- docs/installation.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 25982e6d..18e73b46 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,12 +6,14 @@ Installation ============ -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, supported versions are from 1.0 to 1.7 (see [here](https://www.tensorflow.org/versions/r1.7/install/) for TF 1.7 installation instructions). +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, tested versions are from 1.0 to 1.7 (see here_ for TF installation instructions). + +.. _here: https://www.tensorflow.org/install/ We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. Then simply run -```bash -pip install t3f -``` \ No newline at end of file +.. code-block:: bash + + pip install t3f From c13741ec6c1eea12908be95d62768c5e71a62639 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 27 Oct 2018 19:51:52 +0100 Subject: [PATCH 064/233] test with tf up to 1.11 --- .travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6f27b5df..45f84193 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,12 +8,11 @@ python: env: matrix: - TF_VERSION=1.0 - - TF_VERSION=1.1 - - TF_VERSION=1.2 - - TF_VERSION=1.3 - - TF_VERSION=1.4 - - TF_VERSION=1.6 - TF_VERSION=1.7 + - TF_VERSION=1.8 + - TF_VERSION=1.9 + - TF_VERSION=1.10 + - TF_VERSION=1.11 # command to install dependencies install: # Python 2.7 has very old pip by default that doesn't have tensorflow. From 47c06ed8c006d01b308e9376b8a80f889da9e969 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 27 Oct 2018 20:08:03 +0100 Subject: [PATCH 065/233] init first few words about the library --- docs/index.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 92d4a1f8..48533b16 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,13 @@ -t3f - library for working with tensor train decomposition built on top of TensorFlow +t3f - library for working with Tensor Train decomposition built on top of TensorFlow ============================= -Hi! +t3f is a library for working with Tensor Train decomposition. Tensor Train decomposition is a generalization of the low-rank decomposition from matrices to tensors (=multidimensional arrays), i.e. it's a tool to efficiently work with structured tensors. +t3f is implemented on top of TensorFlow which gives it a few nice properties: + +- GPU support -- just run your model on a machine with a CUDA-enabled GPU and GPU version of the TensorFlow, and t3f will execute most of the operations on it. +- Autodiff -- TensorFlow can automatically compute the derivative of a function with respect to the underlying parameters of the Tensor Train decomposition (TT-cores). Also, if you are into the Riemannian optimization, you can automatically compute the Riemannian gradient of a given function. Don't worry if you don't know what it is :) +- Batch processing -- you can run a single vectorized operation on a set of Tensor Train objects. +- Easy to use with Deep Learning, e.g. you can define a layer parametrized with a Tensor Train object and use it as a part of your favorite neural network implemented in TensorFlow. .. toctree:: :maxdepth: 1 From c13d21993d963477e241d71dc1b475698f72bd60 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 27 Oct 2018 20:19:19 +0100 Subject: [PATCH 066/233] init quick_start --- docs/index.rst | 1 + docs/quick_start.rst | 66 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 docs/quick_start.rst diff --git a/docs/index.rst b/docs/index.rst index 48533b16..f7af091b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,4 +14,5 @@ t3f is implemented on top of TensorFlow which gives it a few nice properties: :caption: Getting started installation + quick_start api \ No newline at end of file diff --git a/docs/quick_start.rst b/docs/quick_start.rst new file mode 100644 index 00000000..7764ecc6 --- /dev/null +++ b/docs/quick_start.rst @@ -0,0 +1,66 @@ +.. t3f documentation master file, created by + sphinx-quickstart on Sun Mar 12 10:06:09 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Quick start +=========== + +Import the libraries + +.. code-block:: python + + import tensorflow as tf + import t3f + +Generate a random tensor and compute its norm. + +.. code-block:: python + + # Create a random tensor of shape (3, 2, 2). + a = t3f.random_tensor((3, 2, 2), tt_rank=3) + norm = t3f.frobenius_norm(a) + # Convert TT-tensor into a dense tensor for printing. + a_full = t3f.full(a) + # Run a tensorflow session to run the operations. + with tf.Session() as sess: + # Run the operations. Note that if you run these + # two operations separetly (sess.run(a_full), sess.run(norm)) + # the result will be different, since sess.run will + # generate a new random tensor a on each run because `a' is + # an operation 'generate me a random tensor'. + a_val, norm_val = sess.run([a_full, norm]) + print('The norm is %f' % norm_val) + print(a_val) + +Arithmetic operations + +.. code-block:: python + + a = t3f.random_tensor((3, 2, 2), tt_rank=3) + b_dense = tf.random_normal((3, 2, 2)) + # Use TT-SVD on b_dense. + b_tt = t3f.to_tt_tensor(b_dense, max_tt_rank=4) + sum_round = t3f.round(t3f.add(a, b_tt), max_tt_rank=2) + +Tensor operations + +.. code-block:: python + + # Inner product (sum of products of all elements). + a = t3f.random_tensor((3, 2, 2), tt_rank=3) + b = t3f.random_tensor((3, 2, 2), tt_rank=4) + inner_prod = t3f.flat_inner(a, b) + +Matrix operations + +.. code-block:: python + + A = t3f.random_matrix(((3, 2, 2), (2, 3, 3)), tt_rank=3) + b = t3f.random_matrix(((2, 3, 3), None), tt_rank=3) + # Matrix-by-vector + matvec = t3f.matmul(A, b) + + # Matrix-by-dense matrix + b_dense = tf.random_normal((18, 1)) + matvec2 = t3f.matmul(A, b_dense) From f375696001ea990980ba45593684fe86ce91ac2d Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 27 Oct 2018 20:23:07 +0100 Subject: [PATCH 067/233] update readme with newer tf versions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35ef70b4..16a89841 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ TensorFlow implementation of the Tensor Train (TT) -Toolbox. API is available via [readthedocs](https://t3f.readthedocs.io/en/latest/). # Installation -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, supported versions are from 1.0 to 1.7 (see [here](https://www.tensorflow.org/versions/r1.7/install/) for TF 1.7 installation instructions). +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, tested versions are from 1.0 to 1.11 (see [here](https://www.tensorflow.org/install/) for TF installation instructions). We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. Then simply run ```bash From 5b6088656e4805e8047ec2cd2d0c1666f74e9f77 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 27 Oct 2018 21:56:53 +0100 Subject: [PATCH 068/233] start testing with different dtypes --- t3f/ops_test.py | 180 +++++++++++++++++++++++++++++------------------- 1 file changed, 108 insertions(+), 72 deletions(-) diff --git a/t3f/ops_test.py b/t3f/ops_test.py index cb1dcc7c..914a736c 100644 --- a/t3f/ops_test.py +++ b/t3f/ops_test.py @@ -8,13 +8,13 @@ from t3f import initializers -class TTTensorTest(tf.test.TestCase): +class _TTTensorTest(): def testFullTensor2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(10, rank).astype(np.float32) - b = np.random.rand(rank, 9).astype(np.float32) + a = np.random.rand(10, rank).astype(self.np_dtype) + b = np.random.rand(rank, 9).astype(self.np_dtype) tt_cores = (a.reshape(1, 10, rank), b.reshape(rank, 9, 1)) desired = np.dot(a, b) with self.test_session(): @@ -25,9 +25,9 @@ def testFullTensor2d(self): def testFullTensor3d(self): np.random.seed(1) for rank_1 in [1, 2]: - a = np.random.rand(10, rank_1).astype(np.float32) - b = np.random.rand(rank_1, 9, 3).astype(np.float32) - c = np.random.rand(3, 8).astype(np.float32) + a = np.random.rand(10, rank_1).astype(self.np_dtype) + b = np.random.rand(rank_1, 9, 3).astype(self.np_dtype) + c = np.random.rand(3, 8).astype(self.np_dtype) tt_cores = (a.reshape(1, 10, rank_1), b, c.reshape((3, 8, 1))) # Basically do full by hand. desired = a.dot(b.reshape((rank_1, -1))) @@ -69,10 +69,11 @@ def testFlatInnerTTTensbySparseTens(self): for rank in rank_list: for num_elements in [1, 10]: tt_1 = initializers.random_tensor(shape, tt_rank=rank) - sparse_flat_indices = np.random.choice(np.prod(shape), num_elements).astype(int) + sparse_flat_indices = np.random.choice(np.prod(shape), num_elements) + sparse_flat_indices = sparse_flat_indices.astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, shape) sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(num_elements).astype(np.float32) + values = np.random.randn(num_elements).astype(self.np_dtype) sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, dense_shape=shape) res_actual = ops.flat_inner(tt_1, sparse_2) @@ -144,11 +145,10 @@ def testCastFloat(self): tt_x = initializers.random_tensor((2, 3, 2), tt_rank=2) with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_x, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) + casted = ops.cast(tt_x, self.tf_dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(self.tf_dtype, casted.dtype) + self.assertTrue(self.tf_dtype, casted_val.dtype) def testCastIntFloat(self): # Tests cast function from int to float for tensors. @@ -159,11 +159,10 @@ def testCastIntFloat(self): tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_int, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) + casted = ops.cast(tt_int, self.tf_dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(self.tf_dtype, casted.dtype) + self.assertTrue(self.tf_dtype, casted_val.dtype) def testCoreRenorm(self): a = initializers.random_tensor(3 * (10,), tt_rank=7) @@ -180,13 +179,13 @@ def testCoreRenorm(self): * np.ones((len(b_cores)))) -class TTMatrixTest(tf.test.TestCase): +class _TTMatrixTest(): def testFullMatrix2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(2, 3, rank).astype(np.float32) - b = np.random.rand(rank, 4, 5).astype(np.float32) + a = np.random.rand(2, 3, rank).astype(self.np_dtype) + b = np.random.rand(rank, 4, 5).astype(self.np_dtype) tt_cores = (a.reshape(1, 2, 3, rank), b.reshape((rank, 4, 5, 1))) # Basically do full by hand. desired = a.reshape((-1, rank)).dot(b.reshape((rank, -1))) @@ -201,9 +200,9 @@ def testFullMatrix2d(self): def testFullMatrix3d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(2, 3, rank).astype(np.float32) - b = np.random.rand(rank, 4, 5, rank).astype(np.float32) - c = np.random.rand(rank, 2, 2).astype(np.float32) + a = np.random.rand(2, 3, rank).astype(self.np_dtype) + b = np.random.rand(rank, 4, 5, rank).astype(self.np_dtype) + c = np.random.rand(rank, 2, 2).astype(self.np_dtype) tt_cores = (a.reshape(1, 2, 3, rank), b.reshape(rank, 4, 5, rank), c.reshape(rank, 2, 2, 1)) # Basically do full by hand. @@ -237,7 +236,7 @@ def testTTMatTimesDenseVec(self): inp_shape = (2, 3, 4) out_shape = (3, 4, 3) np.random.seed(1) - vec = np.random.rand(np.prod(inp_shape), 1).astype(np.float32) + vec = np.random.rand(np.prod(inp_shape), 1).astype(self.np_dtype) with self.test_session() as sess: tf_vec = tf.constant(vec) tf.set_random_seed(1) @@ -252,7 +251,8 @@ def testDenseMatTimesTTVec(self): inp_shape = (3, 3, 3, 3) out_shape = (3, 3, 3, 3) np.random.seed(1) - mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)).astype(np.float32) + mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) + mat = mat.astype(self.np_dtype) with self.test_session() as sess: tf_mat = tf.constant(mat) tf.set_random_seed(1) @@ -296,7 +296,7 @@ def testFlatInnerTTMatbySparseMat(self): sparse_flat_indices = sparse_flat_indices.astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, matrix_shape) sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(num_elements).astype(np.float32) + values = np.random.randn(num_elements).astype(self.np_dtype) sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, dense_shape=matrix_shape) res_actual = ops.flat_inner(tt_1, sparse_2) @@ -383,11 +383,10 @@ def testCastFloat(self): with self.test_session() as sess: for tt in [tt_mat, tt_vec]: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) + casted = ops.cast(tt, self.tf_dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(self.tf_dtype, casted.dtype) + self.assertTrue(self.tf_dtype, casted_val.dtype) def testCastIntFloat(self): # Tests cast function from int to float for matrices. @@ -398,16 +397,15 @@ def testCastIntFloat(self): tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_int, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) + casted = ops.cast(tt_int, self.tf_dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(self.tf_dtype, casted.dtype) + self.assertTrue(self.tf_dtype, casted_val.dtype) def testUnknownRanksTTMatmul(self): # Tests tt_tt_matmul for matrices with unknown ranks - K_1 = tf.placeholder(tf.float32, (1, 2, 2, None)) - K_2 = tf.placeholder(tf.float32, (None, 3, 3, 1)) + K_1 = tf.placeholder(self.tf_dtype, (1, 2, 2, None)) + K_2 = tf.placeholder(self.tf_dtype, (None, 3, 3, 1)) tt_mat = TensorTrain([K_1, K_2]) res_actual = ops.full(ops.matmul(tt_mat, tt_mat)) res_desired = tf.matmul(ops.full(tt_mat), ops.full(tt_mat)) @@ -424,8 +422,8 @@ def testHalfKnownRanksTTMatmul(self): # Tests tt_tt_matmul for the case when one matrice has known ranks # and the other one doesn't np.random.seed(1) - K_1 = tf.placeholder(tf.float32, (1, 2, 2, None)) - K_2 = tf.placeholder(tf.float32, (None, 3, 3, 1)) + K_1 = tf.placeholder(self.tf_dtype, (1, 2, 2, None)) + K_2 = tf.placeholder(self.tf_dtype, (None, 3, 3, 1)) tt_mat_known_ranks = TensorTrain([K_1, K_2], tt_ranks=[1, 3, 1]) tt_mat = TensorTrain([K_1, K_2]) res_actual = ops.full(ops.matmul(tt_mat_known_ranks, tt_mat)) @@ -439,13 +437,13 @@ def testHalfKnownRanksTTMatmul(self): self.assertAllClose(res_desired_val, res_actual_val) -class TTTensorBatchTest(tf.test.TestCase): +class _TTTensorBatchTest(): def testFullTensor2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(3, 10, rank).astype(np.float32) - b = np.random.rand(3, rank, 9).astype(np.float32) + a = np.random.rand(3, 10, rank).astype(self.np_dtype) + b = np.random.rand(3, rank, 9).astype(self.np_dtype) tt_cores = (a.reshape(3, 1, 10, rank), b.reshape(3, rank, 9, 1)) desired = np.einsum('oib,obj->oij', a, b) with self.test_session(): @@ -456,9 +454,9 @@ def testFullTensor2d(self): def testFullTensor3d(self): np.random.seed(1) for rank_1 in [1, 2]: - a = np.random.rand(3, 10, rank_1).astype(np.float32) - b = np.random.rand(3, rank_1, 9, 3).astype(np.float32) - c = np.random.rand(3, 3, 8).astype(np.float32) + a = np.random.rand(3, 10, rank_1).astype(self.np_dtype) + b = np.random.rand(3, rank_1, 9, 3).astype(self.np_dtype) + c = np.random.rand(3, 3, 8).astype(self.np_dtype) tt_cores = (a.reshape(3, 1, 10, rank_1), b, c.reshape((3, 3, 8, 1))) # Basically do full by hand. desired = np.einsum('oia,oajb,obk->oijk', a, b, c) @@ -610,8 +608,8 @@ def testMultiplyBroadcasting(self): self.assertAllClose(res_actual2_val, res_desired_val) def testMultiplyUnknownBatchSizeBroadcasting(self): - c1 = tf.placeholder(tf.float32, [None, 1, 3, 2]) - c2 = tf.placeholder(tf.float32, [None, 2, 3, 1]) + c1 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) + c2 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) tt_a = TensorTrainBatch([c1, c2]) tt_b = initializers.random_tensor_batch((3, 3), tt_rank=3, batch_size=1) tt_c = initializers.random_tensor((3, 3), tt_rank=3) @@ -632,10 +630,10 @@ def testMultiplyUnknownBatchSizeBroadcasting(self): self.assertAllClose(ca, des_ac) def testMultiplyTwoBatchesUnknownSize(self): - c1 = tf.placeholder(tf.float32, [None, 1, 3, 2]) - c2 = tf.placeholder(tf.float32, [None, 2, 3, 1]) - c3 = tf.placeholder(tf.float32, [None, 1, 3, 2]) - c4 = tf.placeholder(tf.float32, [None, 2, 3, 1]) + c1 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) + c2 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) + c3 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) + c4 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) tt_a = TensorTrainBatch([c1, c2]) tt_b = TensorTrainBatch([c3, c4]) res_ab = ops.full(ops.multiply(tt_a, tt_b)) @@ -660,8 +658,8 @@ def testMultiplyTwoBatchesUnknownSize(self): sess.run(to_run, feed_dict=feed_dict_err) def testMultiplyUnknownSizeBatchAndBatch(self): - c1 = tf.placeholder(tf.float32, [None, 1, 3, 2]) - c2 = tf.placeholder(tf.float32, [None, 2, 3, 1]) + c1 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) + c2 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) tt_b = initializers.random_tensor_batch((3, 3), tt_rank=2, batch_size=8) tt_a = TensorTrainBatch([c1, c2]) res_ab = ops.full(ops.multiply(tt_a, tt_b)) @@ -722,13 +720,13 @@ def testCoreRenormBatch(self): self.assertAllClose(b_cores_norms, b_cores_norms[0] * np.ones((len(b_cores)))) -class TTMatrixTestBatch(tf.test.TestCase): +class _TTMatrixTestBatch(): def testFullMatrix2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(3, 2, 3, rank).astype(np.float32) - b = np.random.rand(3, rank, 4, 5).astype(np.float32) + a = np.random.rand(3, 2, 3, rank).astype(self.np_dtype) + b = np.random.rand(3, rank, 4, 5).astype(self.np_dtype) tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape((3, rank, 4, 5, 1))) # Basically do full by hand. desired = np.einsum('oijb,obkl->oijkl', a, b) @@ -743,9 +741,9 @@ def testFullMatrix2d(self): def testFullMatrix3d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(3, 2, 3, rank).astype(np.float32) - b = np.random.rand(3, rank, 4, 5, rank).astype(np.float32) - c = np.random.rand(3, rank, 2, 2).astype(np.float32) + a = np.random.rand(3, 2, 3, rank).astype(self.np_dtype) + b = np.random.rand(3, rank, 4, 5, rank).astype(self.np_dtype) + c = np.random.rand(3, rank, 2, 2).astype(self.np_dtype) tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape(3, rank, 4, 5, rank), c.reshape(3, rank, 2, 2, 1)) # Basically do full by hand. @@ -844,11 +842,10 @@ def testCastFloat(self): batch_size=3) with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_mat, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) + casted = ops.cast(tt_mat, self.tf_dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(self.tf_dtype, casted.dtype) + self.assertTrue(self.tf_dtype, casted_val.dtype) def testCastIntFloat(self): # Tests cast function from int to float for matrices. @@ -860,21 +857,60 @@ def testCastIntFloat(self): tt_int_batch = shapes.expand_batch_dim(tt_int) with self.test_session() as sess: - for dtype in [tf.float16, tf.float32, tf.float64]: - casted = ops.cast(tt_int_batch, dtype) - casted_val = sess.run(ops.full(casted)) - self.assertEqual(dtype, casted.dtype) - self.assertTrue(dtype, casted_val.dtype) + casted = ops.cast(tt_int_batch, self.tf_dtype) + casted_val = sess.run(ops.full(casted)) + self.assertEqual(self.tf_dtype, casted.dtype) + self.assertTrue(self.tf_dtype, casted_val.dtype) def _random_sparse(shape, non_zeros): sparse_flat_indices = np.random.choice(np.prod(shape), non_zeros).astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, shape) sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(non_zeros).astype(np.float32) + values = np.random.randn(non_zeros).astype(self.np_dtype) sparse = tf.SparseTensor(indices=sparse_indices, values=values, dense_shape=shape) return sparse + +class TTTensorTestFloat32(tf.test.TestCase, _TTTensorTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class TTTensorTestFloat64(tf.test.TestCase, _TTTensorTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + +class TTMatrixTestFloat32(tf.test.TestCase, _TTMatrixTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class TTMatrixTestFloat64(tf.test.TestCase, _TTMatrixTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + +class TTTensorBatchTestFloat32(tf.test.TestCase, _TTTensorBatchTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class TTTensorBatchTestFloat64(tf.test.TestCase, _TTTensorBatchTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + +class TTMatrixTestBatchFloat32(tf.test.TestCase, _TTMatrixTestBatch): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class TTMatrixTestBatchFloat64(tf.test.TestCase, _TTMatrixTestBatch): + np_dtype = np.float64 + tf_dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From ca4678ca34f85952537b7295084a05d58a9a6514 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 27 Oct 2018 22:05:13 +0100 Subject: [PATCH 069/233] add dtype arg to initializers --- t3f/initializers.py | 96 +++++++++++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 33 deletions(-) diff --git a/t3f/initializers.py b/t3f/initializers.py index 6f192cc0..7036416c 100644 --- a/t3f/initializers.py +++ b/t3f/initializers.py @@ -69,11 +69,12 @@ def _validate_input_parameters(is_tensor, shape, **params): '1 or %d, got %d' % (shape[0].size + 1, tt_rank.size)) -def tensor_ones(shape): +def tensor_ones(shape, dtype=tf.float32): """Generate TT-tensor of the given shape with all entries equal to 1. Args: shape: array representing the shape of the future tensor + dtype: [tf.float32] dtype of the resulting tensor. Returns: TensorTrain object containing a TT-tensor @@ -87,16 +88,17 @@ def tensor_ones(shape): tt_cores = num_dims * [None] for i in range(num_dims): curr_core_shape = (1, shape[i], 1) - tt_cores[i] = tf.ones(curr_core_shape) + tt_cores[i] = tf.ones(curr_core_shape, dtype=dtype) return TensorTrain(tt_cores, shape, tt_rank) -def tensor_zeros(shape): +def tensor_zeros(shape, dtype=tf.float32): """Generate TT-tensor of the given shape with all entries equal to 0. Args: shape: array representing the shape of the future tensor + dtype: [tf.float32] dtype of the resulting tensor. Returns: TensorTrain object containing a TT-tensor @@ -109,17 +111,18 @@ def tensor_zeros(shape): tt_cores = num_dims * [None] for i in range(num_dims): curr_core_shape = (1, shape[i], 1) - tt_cores[i] = tf.zeros(curr_core_shape) + tt_cores[i] = tf.zeros(curr_core_shape, dtype=dtype) return TensorTrain(tt_cores, shape, tt_rank) -def eye(shape): +def eye(shape, dtype=tf.float32): """Creates an identity TT-matrix. Args: shape: array which defines the shape of the matrix row and column - indices. + indices. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrain containing an identity TT-matrix of size @@ -135,13 +138,13 @@ def eye(shape): tt_cores = num_dims * [None] for i in range(num_dims): curr_core_shape = (1, shape[i], shape[i], 1) - tt_cores[i] = tf.reshape(tf.eye(shape[i]), curr_core_shape) + tt_cores[i] = tf.reshape(tf.eye(shape[i], dtype=dtype), curr_core_shape) true_shape = np.vstack([shape, shape]) return TensorTrain(tt_cores, true_shape, tt_ranks) -def matrix_ones(shape): +def matrix_ones(shape, dtype=tf.float32): """Generate a TT-matrix of the given shape with each entry equal to 1. Args: @@ -153,6 +156,7 @@ def matrix_ones(shape): and matrix_ones([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrain containing a TT-matrix of size @@ -178,12 +182,12 @@ def matrix_ones(shape): tt_cores = [None] * num_dims for i in range(num_dims): curr_core_shape = (1, shape[0][i], shape[1][i], 1) - tt_cores[i] = tf.ones(curr_core_shape) + tt_cores[i] = tf.ones(curr_core_shape, dtype=dtype) return TensorTrain(tt_cores, shape, tt_rank) -def matrix_zeros(shape): +def matrix_zeros(shape, dtype=tf.float32): """Generate a TT-matrix of the given shape with each entry equal to 0. Args: @@ -195,6 +199,7 @@ def matrix_zeros(shape): and matrix_zeros([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrain containing a TT-matrix of size @@ -219,12 +224,13 @@ def matrix_zeros(shape): tt_cores = [None] * num_dims for i in range(num_dims): curr_core_shape = (1, shape[0][i], shape[1][i], 1) - tt_cores[i] = tf.zeros(curr_core_shape) + tt_cores[i] = tf.zeros(curr_core_shape, dtype=dtype) return TensorTrain(tt_cores, shape, tt_rank) -def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): +def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1., + dtype=tf.float32): """Generate a TT-tensor of the given shape with N(mean, stddev^2) cores. Args: @@ -234,6 +240,7 @@ def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): initializing TT-cores. stddev: a number, the standard deviation of the normal distribution used for initializing TT-cores. + dtype: [tf.float32] dtype of the resulting tensor. Returns: TensorTrain containing a TT-tensor @@ -255,13 +262,14 @@ def tensor_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): tt_cores = [None] * num_dims for i in range(num_dims): curr_core_shape = (tt_rank[i], shape[i], tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev, + dtype=dtype) return TensorTrain(tt_cores, shape, tt_rank) def tensor_batch_with_random_cores(shape, tt_rank=2, batch_size=1, - mean=0., stddev=1.): + mean=0., stddev=1., dtype=tf.float32): """Generate a batch of TT-tensors of given shape with N(mean, stddev^2) cores. Args: @@ -272,6 +280,7 @@ def tensor_batch_with_random_cores(shape, tt_rank=2, batch_size=1, initializing TT-cores. stddev: a number, the standard deviation of the normal distribution used for initializing TT-cores. + dtype: [tf.float32] dtype of the resulting tensor. Returns: TensorTrainBatch containing TT-tensors @@ -293,12 +302,14 @@ def tensor_batch_with_random_cores(shape, tt_rank=2, batch_size=1, tt_cores = [None] * num_dims for i in range(num_dims): curr_core_shape = (batch_size, tt_rank[i], shape[i], tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev, + dtype=dtype) return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) -def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): +def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1., + dtype=tf.float32): """Generate a TT-matrix of given shape with N(mean, stddev^2) cores. Args: @@ -315,6 +326,7 @@ def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): initializing TT-cores. stddev: a number, the standard deviation of the normal distribution used for initializing TT-cores. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrain containing a TT-matrix of size @@ -344,13 +356,14 @@ def matrix_with_random_cores(shape, tt_rank=2, mean=0., stddev=1.): for i in range(num_dims): curr_core_shape = (tt_rank[i], shape[0][i], shape[1][i], tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev, + dtype=dtype) return TensorTrain(tt_cores, shape, tt_rank) def matrix_batch_with_random_cores(shape, tt_rank=2, batch_size=1, - mean=0., stddev=1.): + mean=0., stddev=1., dtype=tf.float32): """Generate a batch of TT-matrices of given shape with N(mean, stddev^2) cores. Args: @@ -369,6 +382,7 @@ def matrix_batch_with_random_cores(shape, tt_rank=2, batch_size=1, initializing TT-cores. stddev: a number, the standard deviation of the normal distribution used for initializing TT-cores. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrainBatch containing a batch of TT-matrices of size @@ -398,7 +412,8 @@ def matrix_batch_with_random_cores(shape, tt_rank=2, batch_size=1, for i in range(num_dims): curr_core_shape = (batch_size, tt_rank[i], shape[0][i], shape[1][i], tt_rank[i + 1]) - tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev) + tt_cores[i] = tf.random_normal(curr_core_shape, mean=mean, stddev=stddev, + dtype=dtype) return TensorTrainBatch(tt_cores, shape, tt_rank, batch_size) @@ -451,7 +466,7 @@ def zeros_like(tt): return tensor_zeros(shape[0, :]) -def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): +def random_tensor(shape, tt_rank=2, mean=0., stddev=1., dtype=tf.float32): """Generate a random TT-tensor of the given shape with given mean and stddev. Entries of the generated tensor (in the full format) will be iid and satisfy @@ -470,6 +485,7 @@ def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): mean: a number, the desired mean for the distribution of entries. stddev: a number, the desired standard deviation for the distribution of entries. + dtype: [tf.float32] dtype of the resulting tensor. Returns: TensorTrain containing a TT-tensor @@ -493,7 +509,8 @@ def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): cr_exponent = -1.0 / (2 * num_dims) var = np.prod(tt_rank ** cr_exponent) core_stddev = stddev ** (1.0 / num_dims) * var - tt = tensor_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) + tt = tensor_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev, + dtype=dtype) if np.abs(mean) < 1e-8: return tt @@ -501,7 +518,8 @@ def random_tensor(shape, tt_rank=2, mean=0., stddev=1.): raise NotImplementedError('non-zero mean is not supported yet') -def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): +def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1., + dtype=tf.float32): """Generate a batch of TT-tensors with given shape, mean and stddev. Entries of the generated tensors (in the full format) will be iid and satisfy @@ -521,6 +539,7 @@ def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): mean: a number, the desired mean for the distribution of entries. stddev: a number, the desired standard deviation for the distribution of entries. + dtype: [tf.float32] dtype of the resulting tensor. Returns: TensorTrainBatch containing TT-tensors. @@ -542,7 +561,7 @@ def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): var = np.prod(tt_rank ** cr_exponent) cr_stddev = stddev ** (1.0 / num_dims) * var tt = tensor_batch_with_random_cores(shape, tt_rank=tt_rank, stddev=cr_stddev, - batch_size=batch_size) + batch_size=batch_size, dtype=dtype) if np.abs(mean) < 1e-8: return tt @@ -550,7 +569,7 @@ def random_tensor_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): raise NotImplementedError('non-zero mean is not supported yet') -def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): +def random_matrix(shape, tt_rank=2, mean=0., stddev=1., dtype=tf.float32): """Generate a random TT-matrix of the given shape with given mean and stddev. Entries of the generated matrix (in the full format) will be iid and satisfy @@ -575,6 +594,7 @@ def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): mean: a number, the desired mean for the distribution of entries. stddev: a number, the desired standard deviation for the distribution of entries. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrain containing a TT-matrix of size @@ -609,7 +629,8 @@ def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): cr_exponent = -1.0 / (2 * num_dims) var = np.prod(tt_rank ** cr_exponent) core_stddev = stddev ** (1.0 / num_dims) * var - tt = matrix_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev) + tt = matrix_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev, + dtype=dtype) if np.abs(mean) < 1e-8: return tt @@ -617,7 +638,8 @@ def random_matrix(shape, tt_rank=2, mean=0., stddev=1.): raise NotImplementedError('non-zero mean is not supported yet') -def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): +def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1., + dtype=tf.float32): """Generate a batch of TT-matrices with given shape, mean and stddev. Entries of the generated matrices (in the full format) will be iid and @@ -643,6 +665,7 @@ def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): mean: a number, the desired mean for the distribution of entries. stddev: a number, the desired standard deviation for the distribution of entries. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrainBatch containing a batch of TT-matrices of size @@ -673,7 +696,8 @@ def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): core_stddev = stddev ** (1.0 / num_dims) * var tt = matrix_batch_with_random_cores(shape, tt_rank=tt_rank, stddev=core_stddev, - batch_size=batch_size) + batch_size=batch_size, + dtype=dtype) if np.abs(mean) < 1e-8: return tt @@ -681,7 +705,7 @@ def random_matrix_batch(shape, tt_rank=2, batch_size=1, mean=0., stddev=1.): raise NotImplementedError('non-zero mean is not supported yet') -def glorot_initializer(shape, tt_rank=2): +def glorot_initializer(shape, tt_rank=2, dtype=tf.float32): """Constructs a random TT matrix with entrywise variance 2.0 / (n_in + n_out) Args: @@ -694,6 +718,7 @@ def glorot_initializer(shape, tt_rank=2): glorot_initializer([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. tt_rank: a number or a (d+1)-element array with ranks. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrain containing a TT-matrix of size @@ -715,10 +740,11 @@ def glorot_initializer(shape, tt_rank=2): n_out = np.prod(shape[1]) lamb = 2.0 / (n_in + n_out) - return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) + return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb), + dtype=dtype) -def he_initializer(shape, tt_rank=2): +def he_initializer(shape, tt_rank=2, dtype=tf.float32): """Constructs a random TT matrix with entrywise variance 2.0 / n_in Args: @@ -731,6 +757,7 @@ def he_initializer(shape, tt_rank=2): he_initializer([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. tt_rank: a number or a (d+1)-element array with ranks. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrain containing a TT-matrix of size @@ -751,10 +778,11 @@ def he_initializer(shape, tt_rank=2): n_in = np.prod(shape[0]) lamb = 2.0 / n_in - return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) + return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb), + dtype=dtype) -def lecun_initializer(shape, tt_rank=2): +def lecun_initializer(shape, tt_rank=2, dtype=tf.float32): """Constructs a random TT matrix with entrywise variance 1.0 / n_in Args: @@ -767,6 +795,7 @@ def lecun_initializer(shape, tt_rank=2): lecun_initializer([None, [2, 2, 2]]) will create an 8-element column and row vectors correspondingly. tt_rank: a number or a (d+1)-element array with ranks. + dtype: [tf.float32] dtype of the resulting matrix. Returns: TensorTrain containing a TT-matrix of size @@ -786,4 +815,5 @@ def lecun_initializer(shape, tt_rank=2): _validate_input_parameters(is_tensor=False, shape=shape, tt_rank=tt_rank) n_in = np.prod(shape[0]) lamb = 1.0 / n_in - return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb)) + return random_matrix(shape, tt_rank=tt_rank, stddev=np.sqrt(lamb), + dtype=dtype) From 6aeb2506c32c63095d978510b231a21c87d7ba75 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 27 Oct 2018 22:10:45 +0100 Subject: [PATCH 070/233] begin fixing failing tests --- t3f/ops.py | 4 ++-- t3f/ops_test.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index 6d9e7fed..8091ca9b 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -430,7 +430,7 @@ def tt_sparse_flat_inner(tt_a, sparse_b): a_shape = shapes.lazy_raw_shape(tt_a) a_ranks = shapes.lazy_tt_ranks(tt_a) if tt_a.is_tt_matrix(): - tt_a_elements = tf.ones((num_elements, 1, 1)) + tt_a_elements = tf.ones((num_elements, 1, 1), dtype=tt_a.dtype) # TODO: use t3f.shape is safer?? tensor_shape = tt_a.get_raw_shape() row_idx_linear = tf.cast(sparse_b.indices[:, 0], tf.int64) @@ -1196,7 +1196,7 @@ def renormalize_tt_cores(tt, epsilon=1e-8): return TensorTrain(new_cores) else: sz = (tt.batch_size,) + (len(tt.tt_cores[0].shape) - 1) * (1,) - running_core_log_norms = tf.zeros(sz) + running_core_log_norms = tf.zeros(sz, dtype=tt.dtype) ax = np.arange(len(tt.tt_cores[0].shape))[1:] fact_list = [] for core in tt.tt_cores: diff --git a/t3f/ops_test.py b/t3f/ops_test.py index 914a736c..c0740044 100644 --- a/t3f/ops_test.py +++ b/t3f/ops_test.py @@ -68,7 +68,8 @@ def testFlatInnerTTTensbySparseTens(self): for shape in shape_list: for rank in rank_list: for num_elements in [1, 10]: - tt_1 = initializers.random_tensor(shape, tt_rank=rank) + tt_1 = initializers.random_tensor(shape, tt_rank=rank, + dtype=self.tf_dtype) sparse_flat_indices = np.random.choice(np.prod(shape), num_elements) sparse_flat_indices = sparse_flat_indices.astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, shape) From c2995619b1d62e30e04ded25bf01d8923c9e1adc Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 27 Oct 2018 22:14:08 +0100 Subject: [PATCH 071/233] update tested tf version --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 18e73b46..3f3657ad 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,7 +6,7 @@ Installation ============ -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, tested versions are from 1.0 to 1.7 (see here_ for TF installation instructions). +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, tested versions are from 1.0 to 1.11 (see here_ for TF installation instructions). .. _here: https://www.tensorflow.org/install/ From 16db1a9dddc907ffb1b78e25928084a60fa5abdf Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 18:49:08 +0000 Subject: [PATCH 072/233] explicetly set dtype in all tests --- t3f/ops_test.py | 161 +++++++++++++++++++++++++++++++----------------- 1 file changed, 105 insertions(+), 56 deletions(-) diff --git a/t3f/ops_test.py b/t3f/ops_test.py index c0740044..f0d999b0 100644 --- a/t3f/ops_test.py +++ b/t3f/ops_test.py @@ -47,8 +47,10 @@ def testFlatInnerTTTensbyTTTens(self): with self.test_session() as sess: for shape in shape_list: for rank in rank_list: - tt_1 = initializers.random_tensor(shape, tt_rank=rank) - tt_2 = initializers.random_tensor(shape, tt_rank=rank) + tt_1 = initializers.random_tensor(shape, tt_rank=rank, + dtype=self.tf_dtype) + tt_2 = initializers.random_tensor(shape, tt_rank=rank, + dtype=self.tf_dtype) res_actual = ops.flat_inner(tt_1, tt_2) tt_1_full = tf.reshape(ops.full(tt_1), (1, -1)) tt_2_full = tf.reshape(ops.full(tt_2), (-1, 1)) @@ -84,8 +86,10 @@ def testFlatInnerTTTensbySparseTens(self): def testAdd(self): # Sum two TT-tensors. - tt_a = initializers.random_tensor((2, 1, 3, 4), tt_rank=2) - tt_b = initializers.random_tensor((2, 1, 3, 4), tt_rank=[1, 2, 4, 3, 1]) + tt_a = initializers.random_tensor((2, 1, 3, 4), tt_rank=2, + dtype=self.tf_dtype) + tt_b = initializers.random_tensor((2, 1, 3, 4), tt_rank=[1, 2, 4, 3, 1], + dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_a + tt_b) @@ -97,8 +101,10 @@ def testAdd(self): def testMultiply(self): # Multiply two TT-tensors. - tt_a = initializers.random_tensor((1, 2, 3, 4), tt_rank=2) - tt_b = initializers.random_tensor((1, 2, 3, 4), tt_rank=[1, 1, 4, 3, 1]) + tt_a = initializers.random_tensor((1, 2, 3, 4), tt_rank=2, + dtype=self.tf_dtype) + tt_b = initializers.random_tensor((1, 2, 3, 4), tt_rank=[1, 1, 4, 3, 1], + dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt_a, tt_b)) res_actual2 = ops.full(tt_a * tt_b) @@ -110,7 +116,8 @@ def testMultiply(self): def testMultiplyByNumber(self): # Multiply a tensor by a number. - tt = initializers.random_tensor((1, 2, 3), tt_rank=(1, 2, 3, 1)) + tt = initializers.random_tensor((1, 2, 3), tt_rank=(1, 2, 3, 1), + dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt, 4)) res_actual2 = ops.full(4.0 * tt) @@ -129,7 +136,8 @@ def testFrobeniusNormTens(self): with self.test_session() as sess: for shape in shape_list: for rank in rank_list: - tt = initializers.random_tensor(shape, tt_rank=rank) + tt = initializers.random_tensor(shape, tt_rank=rank, + dtype=self.tf_dtype) norm_sq_actual = ops.frobenius_norm_squared(tt) norm_actual = ops.frobenius_norm(tt) vars = [norm_sq_actual, norm_actual, ops.full(tt)] @@ -166,7 +174,8 @@ def testCastIntFloat(self): self.assertTrue(self.tf_dtype, casted_val.dtype) def testCoreRenorm(self): - a = initializers.random_tensor(3 * (10,), tt_rank=7) + a = initializers.random_tensor(3 * (10,), tt_rank=7, + dtype=self.tf_dtype) b = ops.renormalize_tt_cores(a) var_list = [ops.full(a), ops.full(b)] with self.test_session() as sess: @@ -223,8 +232,10 @@ def testTTMatTimesTTMat(self): sum_shape = (4, 3, 5) right_shape = (4, 4, 4) with self.test_session() as sess: - tt_mat_1 = initializers.random_matrix((left_shape, sum_shape), tt_rank=3) - tt_mat_2 = initializers.random_matrix((sum_shape, right_shape)) + tt_mat_1 = initializers.random_matrix((left_shape, sum_shape), tt_rank=3, + dtype=self.tf_dtype) + tt_mat_2 = initializers.random_matrix((sum_shape, right_shape), + dtype=self.tf_dtype) res_actual = ops.matmul(tt_mat_1, tt_mat_2) res_actual = ops.full(res_actual) res_desired = tf.matmul(ops.full(tt_mat_1), ops.full(tt_mat_2)) @@ -241,7 +252,8 @@ def testTTMatTimesDenseVec(self): with self.test_session() as sess: tf_vec = tf.constant(vec) tf.set_random_seed(1) - tt_mat = initializers.random_matrix((out_shape, inp_shape)) + tt_mat = initializers.random_matrix((out_shape, inp_shape), + dtype=self.tf_dtype) res_actual = ops.matmul(tt_mat, tf_vec) res_desired = tf.matmul(ops.full(tt_mat), tf_vec) res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) @@ -257,7 +269,8 @@ def testDenseMatTimesTTVec(self): with self.test_session() as sess: tf_mat = tf.constant(mat) tf.set_random_seed(1) - tt_vec = initializers.random_matrix((inp_shape, None)) + tt_vec = initializers.random_matrix((inp_shape, None), + dtype=self.tf_dtype) res_actual = ops.matmul(tf_mat, tt_vec) res_desired = tf.matmul(tf_mat, ops.full(tt_vec)) res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) @@ -271,8 +284,10 @@ def testFlatInnerTTMatbyTTMat(self): with self.test_session() as sess: for shape in shape_list: for rank in rank_list: - tt_1 = initializers.random_matrix(shape, tt_rank=rank) - tt_2 = initializers.random_matrix(shape, tt_rank=rank) + tt_1 = initializers.random_matrix(shape, tt_rank=rank, + dtype=self.tf_dtype) + tt_2 = initializers.random_matrix(shape, tt_rank=rank, + dtype=self.tf_dtype) res_actual = ops.flat_inner(tt_1, tt_2) tt_1_full = tf.reshape(ops.full(tt_1), (1, -1)) tt_2_full = tf.reshape(ops.full(tt_2), (-1, 1)) @@ -291,7 +306,8 @@ def testFlatInnerTTMatbySparseMat(self): for tensor_shape in shape_list: for rank in rank_list: for num_elements in [1, 9]: - tt_1 = initializers.random_matrix(tensor_shape, tt_rank=rank) + tt_1 = initializers.random_matrix(tensor_shape, tt_rank=rank, + dtype=self.tf_dtype) matrix_shape = np.prod(tensor_shape[0]), np.prod(tensor_shape[1]) sparse_flat_indices = np.random.choice(np.prod(matrix_shape), num_elements) sparse_flat_indices = sparse_flat_indices.astype(int) @@ -313,7 +329,8 @@ def testFrobeniusNormMatrix(self): with self.test_session() as sess: for tensor_shape in shape_list: for rank in rank_list: - tt = initializers.random_matrix(tensor_shape, tt_rank=rank) + tt = initializers.random_matrix(tensor_shape, tt_rank=rank, + dtype=self.tf_dtype) norm_sq_actual = ops.frobenius_norm_squared(tt) norm_actual = ops.frobenius_norm(tt) vars = [norm_sq_actual, norm_actual, ops.full(tt)] @@ -333,7 +350,8 @@ def testTranspose(self): with self.test_session() as sess: for tensor_shape in shape_list: for rank in rank_list: - tt = initializers.random_matrix(tensor_shape, tt_rank=rank) + tt = initializers.random_matrix(tensor_shape, tt_rank=rank, + dtype=self.tf_dtype) res_actual = ops.full(ops.transpose(tt)) res_actual_val, tt_val = sess.run([res_actual, ops.full(tt)]) self.assertAllClose(tt_val.transpose(), res_actual_val) @@ -346,9 +364,12 @@ def testQuadraticForm(self): with self.test_session() as sess: for tensor_shape in shape_list: for rank in rank_list: - A = initializers.random_matrix(tensor_shape, tt_rank=rank) - b = initializers.random_matrix((tensor_shape[0], None), tt_rank=rank) - c = initializers.random_matrix((tensor_shape[1], None), tt_rank=rank) + A = initializers.random_matrix(tensor_shape, tt_rank=rank, + dtype=self.tf_dtype) + b = initializers.random_matrix((tensor_shape[0], None), tt_rank=rank, + dtype=self.tf_dtype) + c = initializers.random_matrix((tensor_shape[1], None), tt_rank=rank, + dtype=self.tf_dtype) res_actual = ops.quadratic_form(A, b, c) vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] res_actual_val, A_val, b_val, c_val = sess.run(vars) @@ -364,11 +385,14 @@ def testQuadraticFormBatch(self): with self.test_session() as sess: for tensor_shape in shape_list: for rank in rank_list: - A = initializers.random_matrix(tensor_shape, tt_rank=rank) + A = initializers.random_matrix(tensor_shape, tt_rank=rank, + dtype=self.tf_dtype) b = initializers.random_matrix_batch((tensor_shape[0], None), - tt_rank=rank, batch_size=5) + tt_rank=rank, batch_size=5, + dtype=self.tf_dtype) c = initializers.random_matrix_batch((tensor_shape[1], None), - tt_rank=rank, batch_size=5) + tt_rank=rank, batch_size=5, + dtype=self.tf_dtype) res_actual = ops.quadratic_form(A, b, c) vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] res_actual_val, A_val, b_val, c_val = sess.run(vars) @@ -475,9 +499,11 @@ def testFlatInnerTTTensbyTTTensSameBatchSize(self): for shape in shape_list: for rank in rank_list: tt_1 = initializers.random_tensor_batch(shape, tt_rank=rank, - batch_size=2) + batch_size=2, + dtype=self.tf_dtype) tt_2 = initializers.random_tensor_batch(shape, tt_rank=rank, - batch_size=2) + batch_size=2, + dtype=self.tf_dtype) res_actual = ops.flat_inner(tt_1, tt_2) tt_1_full = tf.reshape(ops.full(tt_1), (2, 1, -1)) tt_2_full = tf.reshape(ops.full(tt_2), (2, -1, 1)) @@ -487,8 +513,10 @@ def testFlatInnerTTTensbyTTTensSameBatchSize(self): def testFlatInnerTTTensbyTTTensBroadcasting(self): # Inner product between two batch TT-tensors with broadcasting. - tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=1) - tt_2 = initializers.random_tensor_batch((2, 3, 4), batch_size=3) + tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=1, + dtype=self.tf_dtype) + tt_2 = initializers.random_tensor_batch((2, 3, 4), batch_size=3, + dtype=self.tf_dtype) res_actual_1 = ops.flat_inner(tt_1, tt_2) res_actual_2 = ops.flat_inner(tt_2, tt_1) res_desired = tf.einsum('ijk,oijk->o', ops.full(tt_1[0]), ops.full(tt_2)) @@ -505,9 +533,10 @@ def testFlatInnerTTTensbyTTTensBroadcasting(self): def testAddSameBatchSize(self): # Sum two TT-tensors with the same batch size. - tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=3) + tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=3, + dtype=self.tf_dtype) tt_b = initializers.random_tensor_batch((2, 1, 4), tt_rank=[1, 2, 4, 1], - batch_size=3) + batch_size=3, dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_a + tt_b) @@ -519,9 +548,10 @@ def testAddSameBatchSize(self): def testAddBroadcasting(self): # Sum two TT-tensors with broadcasting. - tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=1) + tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=1, + dtype=self.tf_dtype) tt_b = initializers.random_tensor_batch((2, 1, 4), tt_rank=[1, 2, 4, 1], - batch_size=3) + batch_size=3, dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_b + tt_a) @@ -534,7 +564,7 @@ def testAddBroadcasting(self): def testMultiplyByNumber(self): # Multiply batch of tensors by a number. tt = initializers.random_tensor_batch((1, 2, 3), tt_rank=(1, 2, 3, 1), - batch_size=3) + batch_size=3, dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt, 4)) res_actual2 = ops.full(4.0 * tt) @@ -546,7 +576,8 @@ def testMultiplyByNumber(self): def testFrobeniusNormDifferentiableBatch(self): with self.test_session() as sess: - tt = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) + tt = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, + dtype=self.tf_dtype) norm_sq_diff = ops.frobenius_norm_squared(tt, differentiable=True) variables = [norm_sq_diff, ops.full(tt)] norm_sq_diff_val, tt_full = sess.run(variables) @@ -556,7 +587,8 @@ def testFrobeniusNormDifferentiableBatch(self): def testFrobeniusNormTens(self): # Frobenius norm of a batch of TT-tensors. with self.test_session() as sess: - tt = initializers.tensor_batch_with_random_cores((2, 1, 3), batch_size=3) + tt = initializers.tensor_batch_with_random_cores((2, 1, 3), batch_size=3, + dtype=self.tf_dtype) norm_sq_actual = ops.frobenius_norm_squared(tt) norm_actual = ops.frobenius_norm(tt) vars = [norm_sq_actual, norm_actual, ops.full(tt)] @@ -569,8 +601,9 @@ def testFrobeniusNormTens(self): rtol=1e-5) def testMultiplyBatchByTensor(self): - tt_a = initializers.random_tensor((3, 3, 3), tt_rank=2) - tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) + tt_a = initializers.random_tensor((3, 3, 3), tt_rank=2, dtype=self.tf_dtype) + tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, + dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt_a, tt_b)) res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) @@ -581,8 +614,10 @@ def testMultiplyBatchByTensor(self): self.assertAllClose(res_actual2_val, res_desired_val) def testMultiplyBatchByBatch(self): - tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) - tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) + tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, + dtype=self.tf_dtype) + tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, + dtype=self.tf_dtype) res_actual = ops.full(ops.multiply(tt_a, tt_b)) res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) res_desired = ops.full(tt_a) * ops.full(tt_b) @@ -597,8 +632,10 @@ def testMultiplyBatchByBatch(self): self.assertAllClose(res_actual2_val, res_desired_val) def testMultiplyBroadcasting(self): - tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=1) - tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5) + tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=1, + dtype=self.tf_dtype) + tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, + dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt_a, tt_b)) res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) @@ -612,8 +649,10 @@ def testMultiplyUnknownBatchSizeBroadcasting(self): c1 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) c2 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) tt_a = TensorTrainBatch([c1, c2]) - tt_b = initializers.random_tensor_batch((3, 3), tt_rank=3, batch_size=1) - tt_c = initializers.random_tensor((3, 3), tt_rank=3) + tt_b = initializers.random_tensor_batch((3, 3), tt_rank=3, batch_size=1, + dtype=self.tf_dtype) + tt_c = initializers.random_tensor((3, 3), tt_rank=3, + dtype=self.tf_dtype) res_ab = ops.full(ops.multiply(tt_a, tt_b)) res_ba = ops.full(ops.multiply(tt_b, tt_a)) res_ac = ops.full(ops.multiply(tt_a, tt_c)) @@ -661,7 +700,8 @@ def testMultiplyTwoBatchesUnknownSize(self): def testMultiplyUnknownSizeBatchAndBatch(self): c1 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) c2 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) - tt_b = initializers.random_tensor_batch((3, 3), tt_rank=2, batch_size=8) + tt_b = initializers.random_tensor_batch((3, 3), tt_rank=2, batch_size=8, + dtype=self.tf_dtype) tt_a = TensorTrainBatch([c1, c2]) res_ab = ops.full(ops.multiply(tt_a, tt_b)) res_ba = ops.full(ops.multiply(tt_b, tt_a)) @@ -683,7 +723,7 @@ def testMultiplyUnknownSizeBatchAndBatch(self): def testGatherND(self): idx = [[0, 0, 0], [0, 1, 2], [0, 1, 0]] pl_idx = tf.placeholder(tf.int32, [None, 3]) - tt = initializers.random_tensor((3, 4, 5), tt_rank=2) + tt = initializers.random_tensor((3, 4, 5), tt_rank=2, dtype=self.tf_dtype) res_np = ops.gather_nd(tt, idx) res_pl = ops.gather_nd(tt, pl_idx) res_desired = tf.gather_nd(ops.full(tt), idx) @@ -696,7 +736,8 @@ def testGatherND(self): def testGatherNDBatch(self): idx = [[0, 0, 0, 0], [1, 0, 1, 2], [0, 0, 1, 0]] pl_idx = tf.placeholder(tf.int32, [None, 4]) - tt = initializers.random_tensor_batch((3, 4, 5), tt_rank=2, batch_size=2) + tt = initializers.random_tensor_batch((3, 4, 5), tt_rank=2, batch_size=2, + dtype=self.tf_dtype) res_np = ops.gather_nd(tt, idx) res_pl = ops.gather_nd(tt, pl_idx) res_desired = tf.gather_nd(ops.full(tt), idx) @@ -707,7 +748,8 @@ def testGatherNDBatch(self): self.assertAllClose(res_pl_v, res_pl_v) def testCoreRenormBatch(self): - a = initializers.random_tensor_batch(3 * (10,), tt_rank=7, batch_size=5) + a = initializers.random_tensor_batch(3 * (10,), tt_rank=7, batch_size=5, + dtype=self.tf_dtype) b = ops.renormalize_tt_cores(a) var_list = [ops.full(a), ops.full(b)] @@ -765,9 +807,11 @@ def testTTMatTimesTTMatSameBatchSize(self): right_shape = (4, 4) with self.test_session() as sess: tt_mat_1 = initializers.random_matrix_batch((left_shape, sum_shape), - tt_rank=3, batch_size=3) + tt_rank=3, batch_size=3, + dtype=self.tf_dtype) tt_mat_2 = initializers.random_matrix_batch((sum_shape, right_shape), - batch_size=3) + batch_size=3, + dtype=self.tf_dtype) res_actual = ops.matmul(tt_mat_1, tt_mat_2) res_actual = ops.full(res_actual) res_desired = tf.matmul(ops.full(tt_mat_1), ops.full(tt_mat_2)) @@ -783,8 +827,10 @@ def testTTMatTimesTTMatBroadcasting(self): right_shape = (4, 4) with self.test_session() as sess: tt_mat_1 = initializers.random_matrix_batch((left_shape, sum_shape), - tt_rank=3, batch_size=3) - tt_mat_2 = initializers.random_matrix_batch((sum_shape, right_shape)) + tt_rank=3, batch_size=3, + dtype=self.tf_dtype) + tt_mat_2 = initializers.random_matrix_batch((sum_shape, right_shape), + dtype=self.tf_dtype) # TT-batch by one element TT-batch res_actual = ops.matmul(tt_mat_1, tt_mat_2) res_actual = ops.full(res_actual) @@ -802,7 +848,8 @@ def testTTMatTimesTTMatBroadcasting(self): def testTranspose(self): # Transpose a batch of TT-matrices. with self.test_session() as sess: - tt = initializers.random_matrix_batch(((2, 3, 4), (2, 2, 2)), batch_size=2) + tt = initializers.random_matrix_batch(((2, 3, 4), (2, 2, 2)), + batch_size=2, dtype=self.tf_dtype) res_actual = ops.full(ops.transpose(tt)) res_actual_val, tt_val = sess.run([res_actual, ops.full(tt)]) self.assertAllClose(tt_val.transpose((0, 2, 1)), res_actual_val) @@ -810,9 +857,10 @@ def testTranspose(self): def testAddSameBatchSize(self): # Sum two TT-matrices with the same batch size. tt_a = initializers.random_matrix_batch(((2, 1, 4), None), tt_rank=2, - batch_size=3) + batch_size=3, dtype=self.tf_dtype) tt_b = initializers.random_matrix_batch(((2, 1, 4), None), - tt_rank=[1, 2, 4, 1], batch_size=3) + tt_rank=[1, 2, 4, 1], batch_size=3, + dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_a + tt_b) @@ -825,9 +873,10 @@ def testAddSameBatchSize(self): def testAddBroadcasting(self): # Sum two TT-matrices with broadcasting. tt_a = initializers.random_matrix_batch(((2, 1, 4), (2, 2, 2)), tt_rank=2, - batch_size=3) + batch_size=3, dtype=self.tf_dtype) tt_b = initializers.random_matrix_batch(((2, 1, 4), (2, 2, 2)), - tt_rank=[1, 2, 4, 1], batch_size=1) + tt_rank=[1, 2, 4, 1], batch_size=1, + dtype=self.tf_dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_b + tt_a) From 784cba3757274266d850f4b16b1b7d340103cb85 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 18:49:59 +0000 Subject: [PATCH 073/233] dtype bug fix: convert constant to appropriate dtype before using in renormilize_tt_cores --- t3f/ops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/t3f/ops.py b/t3f/ops.py index 8091ca9b..97ec63b9 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -1179,6 +1179,7 @@ def renormalize_tt_cores(tt, epsilon=1e-8): case applies to each TT in `TensorTrainBatch`. """ + epsilon = tf.cast(epsilon, tt.dtype) if isinstance(tt, TensorTrain): new_cores = [] running_log_norm = 0 From 21ccf41f2e42a8656fde64d41e6e3699c95d2e48 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 18:56:46 +0000 Subject: [PATCH 074/233] test with different dtypes --- t3f/approximate_test.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/t3f/approximate_test.py b/t3f/approximate_test.py index 52a99320..53e492db 100644 --- a/t3f/approximate_test.py +++ b/t3f/approximate_test.py @@ -6,13 +6,14 @@ from t3f import initializers -class ApproximateTest(tf.test.TestCase): +class _ApproximateTest(): def testAddN(self): # Sum a bunch of TT-matrices. - tt_a = initializers.random_matrix(((2, 1, 4), (2, 2, 2)), tt_rank=2) + tt_a = initializers.random_matrix(((2, 1, 4), (2, 2, 2)), tt_rank=2, + dtype=self.tf_dtype) tt_b = initializers.random_matrix(((2, 1, 4), (2, 2, 2)), - tt_rank=[1, 2, 4, 1]) + tt_rank=[1, 2, 4, 1], dtype=self.tf_dtype) def desired(tt_objects): res = tt_objects[0] @@ -48,7 +49,8 @@ def desired(tt_batch): with self.test_session() as sess: tt_batch = initializers.random_tensor_batch((4, 3, 5), tt_rank=2, - batch_size=batch_size) + batch_size=batch_size, + dtype=self.tf_dtype) res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 10)) res_desired = ops.full(desired(tt_batch)) res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) @@ -65,7 +67,8 @@ def desired(tt_batch, coef): with self.test_session() as sess: tt_batch = initializers.random_tensor_batch((4, 3, 5), tt_rank=3, - batch_size=3) + batch_size=3, + dtype=self.tf_dtype) res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 9, [1.2, -0.2, 1])) res_desired = ops.full(desired(tt_batch, [1.2, -0.2, 1])) @@ -81,13 +84,13 @@ def desired(tt_batch, coef): res += coef[i] * tt_batch[i] return res with self.test_session() as sess: - tt_batch = initializers.random_tensor_batch((4, 3, 5), - tt_rank=2, - batch_size=3) + tt_batch = initializers.random_tensor_batch((4, 3, 5), tt_rank=2, + batch_size=3, + dtype=self.tf_dtype) coef = [[1., 0.1], [0.9, -0.2], [0.3, 0.3]] - coef = np.array(coef).astype(np.float32) + coef = np.array(coef).astype(self.np_dtype) res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 6, coef)) res_desired_1 = ops.full(desired(tt_batch, coef[:, 0])) @@ -97,5 +100,15 @@ def desired(tt_batch, coef): self.assertAllClose(res_desired_val, res_actual_val, atol=1e-5, rtol=1e-5) +class ApproximateTestFloat32(tf.test.TestCase, _ApproximateTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class ApproximateTestFloat64(tf.test.TestCase, _ApproximateTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From 1bd939544f9c112f5081a8c0fc7d5b10435bfea0 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 18:57:09 +0000 Subject: [PATCH 075/233] fix dtype error: converte coefs into appropriate dtype --- t3f/approximate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/t3f/approximate.py b/t3f/approximate.py index 4cb8f495..3c560ee6 100644 --- a/t3f/approximate.py +++ b/t3f/approximate.py @@ -73,6 +73,7 @@ def reduce_sum_batch(tt_batch, max_tt_rank, coef=None): is_batch_output = False if coef is not None: coef = tf.convert_to_tensor(coef) + coef = tf.cast(coef, tt_batch.dtype) if len(coef.get_shape()) == 1: tt_batch = batch_ops.multiply_along_batch_dim(tt_batch, coef) elif len(coef.get_shape()) == 2: From ae6c28ee3ebe52736006f5ea48a2660e8106ba41 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 19:04:05 +0000 Subject: [PATCH 076/233] test batch_ops with different dtypes --- t3f/batch_ops_test.py | 61 +++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/t3f/batch_ops_test.py b/t3f/batch_ops_test.py index 4c8cfd17..61651019 100644 --- a/t3f/batch_ops_test.py +++ b/t3f/batch_ops_test.py @@ -6,13 +6,16 @@ from t3f import initializers -class BatchOpsTest(tf.test.TestCase): +class _BatchOpsTest(): def testConcatMatrix(self): # Test concating TTMatrix batches along batch dimension. - first = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=1) - second = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=4) - third = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3) + first = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=1, + dtype=self.tf_dtype) + second = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=4, + dtype=self.tf_dtype) + third = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3, + dtype=self.tf_dtype) first_res = batch_ops.concat_along_batch_dim((first)) first_res = ops.full(first_res) first_second_res = batch_ops.concat_along_batch_dim((first, second)) @@ -45,7 +48,8 @@ def testConcatMatrix(self): def testConcatTensorPlaceholders(self): # Test concating TTTensors of unknown batch sizes along batch dimension. number_of_objects = tf.placeholder(tf.int32) - all = initializers.random_tensor_batch((2, 3), batch_size=5) + all = initializers.random_tensor_batch((2, 3), batch_size=5, + dtype=self.tf_dtype) actual = batch_ops.concat_along_batch_dim((all[:number_of_objects], all[number_of_objects:])) with self.test_session() as sess: @@ -56,7 +60,8 @@ def testConcatTensorPlaceholders(self): def testConcatMatrixPlaceholders(self): # Test concating TTMatrices of unknown batch sizes along batch dimension. number_of_objects = tf.placeholder(tf.int32) - all = initializers.random_matrix_batch(((2, 3), (2, 3)), batch_size=5) + all = initializers.random_matrix_batch(((2, 3), (2, 3)), batch_size=5, + dtype=self.tf_dtype) actual = batch_ops.concat_along_batch_dim((all[:number_of_objects], all[number_of_objects:])) with self.test_session() as sess: @@ -66,7 +71,8 @@ def testConcatMatrixPlaceholders(self): def testBatchMultiply(self): # Test multiplying batch of TTMatrices by individual numbers. - tt = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3) + tt = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3, + dtype=self.tf_dtype) weights = [0.1, 0, -10] actual = batch_ops.multiply_along_batch_dim(tt, weights) individual_desired = [weights[i] * tt[i:i+1] for i in range(3)] @@ -77,7 +83,8 @@ def testBatchMultiply(self): def testGramMatrix(self): # Test Gram Matrix of a batch of TT vectors. - tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=5) + tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=5, + dtype=self.tf_dtype) res_actual = batch_ops.gram_matrix(tt_vectors) full_vectors = tf.reshape(ops.full(tt_vectors), (5, 6)) res_desired = tf.matmul(full_vectors, tf.transpose(full_vectors)) @@ -90,8 +97,9 @@ def testGramMatrixWithMatrix(self): # Test Gram Matrix of a batch of TT vectors with providing a matrix, so we # should compute # res[i, j] = tt_vectors[i] ^ T * matrix * tt_vectors[j] - tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=4) - matrix = initializers.random_matrix(((2, 3), (2, 3))) + tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=4, + dtype=self.tf_dtype) + matrix = initializers.random_matrix(((2, 3), (2, 3)), dtype=self.tf_dtype) res_actual = batch_ops.gram_matrix(tt_vectors, matrix) full_vectors = tf.reshape(ops.full(tt_vectors), (4, 6)) with self.test_session() as sess: @@ -107,8 +115,10 @@ def testGramMatrixWithMatrix(self): def testPairwiseFlatInnerTensor(self): # Test pairwise_flat_inner of a batch of TT tensors. - tt_tensors_1 = initializers.random_tensor_batch((2, 3, 2), batch_size=5) - tt_tensors_2 = initializers.random_tensor_batch((2, 3, 2), batch_size=5) + tt_tensors_1 = initializers.random_tensor_batch((2, 3, 2), batch_size=5, + dtype=self.tf_dtype) + tt_tensors_2 = initializers.random_tensor_batch((2, 3, 2), batch_size=5, + dtype=self.tf_dtype) res_actual = batch_ops.pairwise_flat_inner(tt_tensors_1, tt_tensors_2) full_tensors_1 = tf.reshape(ops.full(tt_tensors_1), (5, 12)) full_tensors_2 = tf.reshape(ops.full(tt_tensors_2), (5, 12)) @@ -121,9 +131,11 @@ def testPairwiseFlatInnerTensor(self): def testPairwiseFlatInnerMatrix(self): # Test pairwise_flat_inner of a batch of TT matrices. tt_vectors_1 = initializers.random_matrix_batch(((2, 3), (2, 3)), - batch_size=5) + batch_size=5, + dtype=self.tf_dtype) tt_vectors_2 = initializers.random_matrix_batch(((2, 3), (2, 3)), - batch_size=5) + batch_size=5, + dtype=self.tf_dtype) res_actual = batch_ops.pairwise_flat_inner(tt_vectors_1, tt_vectors_2) full_vectors_1 = tf.reshape(ops.full(tt_vectors_1), (5, 36)) full_vectors_2 = tf.reshape(ops.full(tt_vectors_2), (5, 36)) @@ -137,9 +149,13 @@ def testPairwiseFlatInnerVectorsWithMatrix(self): # Test pairwise_flat_inner of a batch of TT vectors with providing a matrix, # so we should compute # res[i, j] = tt_vectors[i] ^ T * matrix * tt_vectors[j] - tt_vectors_1 = initializers.random_matrix_batch(((2, 3), None), batch_size=2) - tt_vectors_2 = initializers.random_matrix_batch(((2, 3), None), batch_size=3) - matrix = initializers.random_matrix(((2, 3), (2, 3))) + tt_vectors_1 = initializers.random_matrix_batch(((2, 3), None), + batch_size=2, + dtype=self.tf_dtype) + tt_vectors_2 = initializers.random_matrix_batch(((2, 3), None), + batch_size=3, + dtype=self.tf_dtype) + matrix = initializers.random_matrix(((2, 3), (2, 3)), dtype=self.tf_dtype) res_actual = batch_ops.pairwise_flat_inner(tt_vectors_1, tt_vectors_2, matrix) full_vectors_1 = tf.reshape(ops.full(tt_vectors_1), (2, 6)) @@ -156,6 +172,17 @@ def testPairwiseFlatInnerVectorsWithMatrix(self): res_desired_val[i, j] = curr_val self.assertAllClose(res_desired_val, res_actual_val) + +class BatchOpsTestFloat32(tf.test.TestCase, _BatchOpsTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class BatchOpsTestFloat64(tf.test.TestCase, _BatchOpsTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From d7227d5fe4933bd7fe04937de12d62ad1fefb01f Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 19:04:50 +0000 Subject: [PATCH 077/233] fix dtype bug: convert weight into appropriate dtype --- t3f/batch_ops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/t3f/batch_ops.py b/t3f/batch_ops.py index bd89bb36..d1cf7ebf 100644 --- a/t3f/batch_ops.py +++ b/t3f/batch_ops.py @@ -61,6 +61,7 @@ def multiply_along_batch_dim(batch_tt, weights): TensorTrainBatch """ weights = tf.convert_to_tensor(weights) + weights = tf.cast(weights, batch_tt.dtype) tt_cores = list(batch_tt.tt_cores) if batch_tt.is_tt_matrix(): weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis, tf.newaxis] From ac7a2ee7dcf3ce61ea1c9ea5b9ea1f94172c9949 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 19:05:13 +0000 Subject: [PATCH 078/233] pep8: forgotten blank line --- t3f/ops_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/t3f/ops_test.py b/t3f/ops_test.py index f0d999b0..e8891311 100644 --- a/t3f/ops_test.py +++ b/t3f/ops_test.py @@ -952,6 +952,7 @@ class TTTensorBatchTestFloat64(tf.test.TestCase, _TTTensorBatchTest): np_dtype = np.float64 tf_dtype = tf.float64 + class TTMatrixTestBatchFloat32(tf.test.TestCase, _TTMatrixTestBatch): np_dtype = np.float32 tf_dtype = tf.float32 From 77022dca767a7017aca5a7eab30551c309459781 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 19:09:33 +0000 Subject: [PATCH 079/233] decompositions: test for different dtypes --- t3f/decompositions_test.py | 56 +++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/t3f/decompositions_test.py b/t3f/decompositions_test.py index 5848cdaf..3029198a 100644 --- a/t3f/decompositions_test.py +++ b/t3f/decompositions_test.py @@ -7,12 +7,12 @@ from t3f import initializers -class DecompositionsTest(tf.test.TestCase): +class _DecompositionsTest(): def testTTTensor(self): shape = (2, 1, 4, 3) np.random.seed(1) - tens = np.random.rand(*shape).astype(np.float32) + tens = np.random.rand(*shape).astype(self.np_dtype) tf_tens = tf.constant(tens) tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=3) with self.test_session(): @@ -22,7 +22,7 @@ def testTTTensor(self): self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) # Try to decompose the same tensor with unknown shape. - tf_tens_pl = tf.placeholder(tf.float32, (None, None, 4, None)) + tf_tens_pl = tf.placeholder(self.tf_dtype, (None, None, 4, None)) tt_tens = decompositions.to_tt_tensor(tf_tens_pl, max_tt_rank=3) tt_val = ops.full(tt_tens).eval({tf_tens_pl: tens}) self.assertAllClose(tens, tt_val) @@ -33,8 +33,8 @@ def testTTTensorSimple(self): # Test that a tensor of ones and of zeros can be converted into TT with # TT-rank 1. shape = (2, 1, 4, 3) - tens_arr = (np.zeros(shape).astype(np.float32), - np.ones(shape).astype(np.float32)) + tens_arr = (np.zeros(shape).astype(self.np_dtype), + np.ones(shape).astype(self.np_dtype)) for tens in tens_arr: tf_tens = tf.constant(tens) tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=1) @@ -45,7 +45,7 @@ def testTTTensorSimple(self): self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) # Try to decompose the same tensor with unknown shape. - tf_tens_pl = tf.placeholder(tf.float32, (None, None, None, None)) + tf_tens_pl = tf.placeholder(self.tf_dtype, (None, None, None, None)) tt_tens = decompositions.to_tt_tensor(tf_tens_pl, max_tt_rank=1) tt_val = ops.full(tt_tens).eval({tf_tens_pl: tens}) self.assertAllClose(tens, tt_val) @@ -56,7 +56,7 @@ def testTTVector(self): vec_shape = (2, 1, 4, 3) np.random.seed(1) rows = np.prod(vec_shape) - vec = np.random.rand(rows, 1).astype(np.float32) + vec = np.random.rand(rows, 1).astype(self.np_dtype) tf_vec = tf.constant(vec) tt_vec = decompositions.to_tt_matrix(tf_vec, (vec_shape, None)) with self.test_session(): @@ -66,7 +66,7 @@ def testTTCompositeRankTensor(self): # Test if a composite rank (list of ranks) can be used for decomposition # for tensor. np.random.seed(1) - np_tensor = np.random.rand(2, 3, 3, 1) + np_tensor = np.random.rand(2, 3, 3, 1).astype(self.np_dtype) tf_tensor = tf.constant(np_tensor) tt_ranks = [1, 2, 3, 3, 1] @@ -81,7 +81,7 @@ def testTTCompositeRankMatrix(self): out_shape = (1, 2, 2, 1) np.random.seed(1) mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) - mat = mat.astype(np.float32) + mat = mat.astype(self.np_dtype) tf_mat = tf.constant(mat) tt_ranks = [10, 20, 30, 40, 30] tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), @@ -96,7 +96,7 @@ def testTTMatrix(self): out_shape = (3, 3, 2, 3) np.random.seed(1) mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) - mat = mat.astype(np.float32) + mat = mat.astype(self.np_dtype) tf_mat = tf.constant(mat) tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), max_tt_rank=90) @@ -107,7 +107,8 @@ def testTTMatrix(self): def testRoundTensor(self): shape = (2, 1, 4, 3, 3) np.random.seed(1) - tens = initializers.random_tensor(shape, tt_rank=15) + tens = initializers.random_tensor(shape, tt_rank=15, + dtype=self.tf_dtype) rounded_tens = decompositions.round(tens, max_tt_rank=9) with self.test_session() as sess: vars = [ops.full(tens), ops.full(rounded_tens)] @@ -121,7 +122,8 @@ def testOrthogonalizeLeftToRight(self): shape = (2, 4, 3, 3) tt_ranks = (1, 5, 2, 17, 1) updated_tt_ranks = (1, 2, 2, 6, 1) - tens = initializers.random_tensor(shape, tt_rank=tt_ranks) + tens = initializers.random_tensor(shape, tt_rank=tt_ranks, + dtype=self.tf_dtype) orthogonal = decompositions.orthogonalize_tt_cores(tens) with self.test_session() as sess: tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) @@ -142,7 +144,8 @@ def testOrthogonalizeRightToLeft(self): shape = (2, 4, 3, 3) tt_ranks = (1, 5, 2, 17, 1) updated_tt_ranks = (1, 5, 2, 3, 1) - tens = initializers.random_tensor(shape, tt_rank=tt_ranks) + tens = initializers.random_tensor(shape, tt_rank=tt_ranks, + dtype=self.tf_dtype) orthogonal = decompositions.orthogonalize_tt_cores(tens, left_to_right=False) with self.test_session() as sess: tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) @@ -160,14 +163,14 @@ def testOrthogonalizeRightToLeft(self): should_be_eye_val) -class DecompositionsBatchTest(tf.test.TestCase): +class _DecompositionsBatchTest(): def testOrthogonalizeLeftToRight(self): shape = (2, 4, 3, 3) tt_ranks = (1, 5, 2, 17, 1) updated_tt_ranks = (1, 2, 2, 6, 1) tens = initializers.random_tensor_batch(shape, tt_rank=tt_ranks, - batch_size=2) + batch_size=2, dtype=self.tf_dtype) orthogonal = decompositions.orthogonalize_tt_cores(tens) with self.test_session() as sess: tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) @@ -187,7 +190,8 @@ def testOrthogonalizeLeftToRight(self): def testRoundTensor(self): shape = (2, 1, 4, 3, 3) - tens = initializers.random_tensor_batch(shape, tt_rank=15, batch_size=3) + tens = initializers.random_tensor_batch(shape, tt_rank=15, batch_size=3, + dtype=self.tf_dtype) rounded_tens = decompositions.round(tens, max_tt_rank=9) with self.test_session() as sess: vars = [ops.full(tens), ops.full(rounded_tens)] @@ -199,5 +203,25 @@ def testRoundTensor(self): self.assertAllEqual([1, 2, 2, 8, 3, 1], dynamic_tt_ranks) +class DecompositionsTestFloat32(tf.test.TestCase, _DecompositionsTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class DecompositionsTestFloat64(tf.test.TestCase, _DecompositionsTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + +class DecompositionsBatchTestFloat32(tf.test.TestCase, _DecompositionsBatchTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class DecompositionsBatchTestFloat64(tf.test.TestCase, _DecompositionsBatchTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From d2e4255c881fc0b2da56b8fba015e3316d9a8326 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 19:49:45 +0000 Subject: [PATCH 080/233] initializers: test dtypes --- t3f/initializers_test.py | 91 ++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/t3f/initializers_test.py b/t3f/initializers_test.py index a555eeba..23f69cd5 100644 --- a/t3f/initializers_test.py +++ b/t3f/initializers_test.py @@ -5,19 +5,21 @@ from t3f import ops -class InitializersTest(tf.test.TestCase): +class _InitializersTest(): def testTensorOnesAndZeros(self): - tt_ones = initializers.tensor_ones([2, 3, 4]) - tt_zeros = initializers.tensor_zeros([2, 3, 4]) + tt_ones = initializers.tensor_ones([2, 3, 4], dtype=self.tf_dtype) + tt_zeros = initializers.tensor_zeros([2, 3, 4], dtype=self.tf_dtype) - ones_desired = np.ones((2, 3, 4)) - zeros_desired = np.zeros((2, 3, 4)) + ones_desired = np.ones((2, 3, 4), dtype=self.np_dtype) + zeros_desired = np.zeros((2, 3, 4), dtype=self.np_dtype) with self.test_session() as sess: tt_ones_full = sess.run(ops.full(tt_ones)) tt_zeros_full = sess.run(ops.full(tt_zeros)) self.assertAllClose(tt_ones_full, ones_desired) + self.assertEqual(tt_ones_full.dtype, ones_desired.dtype) self.assertAllClose(tt_zeros_full, zeros_desired) + self.assertEqual(tt_zeros_full.dtype, zeros_desired.dtype) bad_shapes = [[[2, 3]], [-1, 3], [0.1, 4]] for shape in bad_shapes: with self.assertRaises(ValueError): @@ -26,11 +28,13 @@ def testTensorOnesAndZeros(self): initializers.tensor_zeros(shape) def testMatrixOnesAndZeros(self): - tt_ones = initializers.matrix_ones([[2, 3, 4], [1, 2, 5]]) - tt_zeros = initializers.matrix_zeros([[2, 3, 4], [1, 2, 5]]) + tt_ones = initializers.matrix_ones([[2, 3, 4], [1, 2, 5]], + dtype=self.tf_dtype) + tt_zeros = initializers.matrix_zeros([[2, 3, 4], [1, 2, 5]], + dtype=self.tf_dtype) - ones_desired = np.ones((24, 10)) - zeros_desired = np.zeros((24, 10)) + ones_desired = np.ones((24, 10), dtype=self.np_dtype) + zeros_desired = np.zeros((24, 10), dtype=self.np_dtype) bad_shapes = [[[-1, 2, 3], [3, 4, 6]], [[1.5, 2, 4], [2, 5, 6]], [[1], [2, 3]], [2, 3, 4]] @@ -38,7 +42,9 @@ def testMatrixOnesAndZeros(self): tt_ones_full = sess.run(ops.full(tt_ones)) tt_zeros_full = sess.run(ops.full(tt_zeros)) self.assertAllClose(tt_ones_full, ones_desired) + self.assertEqual(tt_ones_full.dtype, ones_desired.dtype) self.assertAllClose(tt_zeros_full, zeros_desired) + self.assertEqual(tt_zeros_full.dtype, zeros_desired.dtype) for shape in bad_shapes: with self.assertRaises(ValueError): initializers.matrix_ones(shape) @@ -46,7 +52,7 @@ def testMatrixOnesAndZeros(self): initializers.matrix_zeros(shape) def testEye(self): - tt_eye = initializers.eye([4, 5, 6]) + tt_eye = initializers.eye([4, 5, 6], dtype=self.tf_dtype) eye_desired = np.eye(120) with self.test_session() as sess: eye_full = sess.run(ops.full(tt_eye)) @@ -57,35 +63,36 @@ def testEye(self): initializers.eye(shape) def testOnesLikeAndZerosLike(self): - a = initializers.random_tensor([2, 3, 4]) + a = initializers.random_tensor([2, 3, 4], dtype=self.tf_dtype) b = initializers.ones_like(a) c = initializers.zeros_like(a) var_list = [ops.full(b), ops.full(c)] with self.test_session() as sess: bf, cf = sess.run(var_list) self.assertAllClose(bf, np.ones((2, 3, 4))) + self.assertEqual(self.np_dtype, bf.dtype) self.assertAllClose(cf, np.zeros((2, 3, 4))) + self.assertEqual(self.np_dtype, cf.dtype) with self.assertRaises(ValueError): initializers.ones_like(1) with self.assertRaises(ValueError): initializers.zeros_like(1) - def testRandomTensor(self): + def testInvalidShapeOrRanksTensor(self): shapes = [[3, 4], [3, 4], [3, 4], [3, 4], [1, -2], [1.1, 2], [[3, 4]]] tt_ranks = [-2, 1.5, [2, 3, 4, 5], [1.5], 2, 2, 2] bad_cases = zip(shapes, tt_ranks) for case in bad_cases: with self.assertRaises(ValueError): - initializers.random_tensor(case[0], tt_rank=case[1]) + initializers.random_tensor(case[0], tt_rank=case[1], + dtype=self.tf_dtype) for case in bad_cases: with self.assertRaises(ValueError): - initializers.tensor_with_random_cores(case[0], tt_rank=case[1]) + initializers.tensor_with_random_cores(case[0], tt_rank=case[1], + dtype=self.tf_dtype) - with self.assertRaises(NotImplementedError): - initializers.random_tensor([1, 2], mean=1.0) - - def testRandomMatrix(self): + def testInvalidShapeOrRanksMatrix(self): shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], @@ -94,14 +101,14 @@ def testRandomMatrix(self): bad_cases = zip(shapes, tt_ranks) for case in bad_cases: with self.assertRaises(ValueError): - initializers.random_matrix(case[0], tt_rank=case[1]) + initializers.random_matrix(case[0], tt_rank=case[1], + dtype=self.tf_dtype) for case in bad_cases: with self.assertRaises(ValueError): - initializers.matrix_with_random_cores(case[0], tt_rank=case[1]) - with self.assertRaises(NotImplementedError): - initializers.random_matrix([[2, 3, 4], [1, 2, 3]], mean=1.0) + initializers.matrix_with_random_cores(case[0], tt_rank=case[1], + dtype=self.tf_dtype) - def testRandomTensorBatch(self): + def testInvalidShapeOrRanksTensorBatch(self): shapes = [[3, 4], [3, 4], [3, 4], [3, 4], [1, -2], [1.1, 2], [[3, 4]], [1, 2], [3, 4]] tt_ranks = [-2, 1.5, [2, 3, 4, 5], [1.5], 2, 2, 2, 2, 2] @@ -110,15 +117,15 @@ def testRandomTensorBatch(self): for case in bad_cases: with self.assertRaises(ValueError): initializers.random_tensor_batch(case[0], tt_rank=case[1], - batch_size=case[2]) + batch_size=case[2], + dtype=self.tf_dtype) for case in bad_cases: with self.assertRaises(ValueError): initializers.tensor_batch_with_random_cores(case[0], tt_rank=case[1], - batch_size=case[2]) - with self.assertRaises(NotImplementedError): - initializers.random_tensor_batch([1, 2, 3], mean=1.0) + batch_size=case[2], + dtype=self.tf_dtype) - def testRandomMatrixBatch(self): + def testInvalidShapeOrRanksMatrixBatch(self): shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], @@ -130,15 +137,15 @@ def testRandomMatrixBatch(self): for case in bad_cases: with self.assertRaises(ValueError): initializers.random_matrix_batch(case[0], tt_rank=case[1], - batch_size=case[2]) + batch_size=case[2], + dtype=self.tf_dtype) for case in bad_cases: with self.assertRaises(ValueError): initializers.matrix_batch_with_random_cores(case[0], tt_rank=case[1], - batch_size=case[2]) - with self.assertRaises(NotImplementedError): - initializers.random_matrix_batch([[1, 2, 3], [1, 2, 3]], mean=1.0) + batch_size=case[2], + dtype=self.tf_dtype) - def testGlorot(self): + def testInvalidShapeOrRanksGlorot(self): shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], @@ -158,9 +165,10 @@ def testHe(self): bad_cases = zip(shapes, tt_ranks) for case in bad_cases: with self.assertRaises(ValueError): - initializers.he_initializer(case[0], tt_rank=case[1]) + initializers.he_initializer(case[0], tt_rank=case[1], + dtype=self.tf_dtype) - def testLecun(self): + def testInvalidShapeOrRanksLecun(self): shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], [[0.5, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]], @@ -169,7 +177,18 @@ def testLecun(self): bad_cases = zip(shapes, tt_ranks) for case in bad_cases: with self.assertRaises(ValueError): - initializers.lecun_initializer(case[0], tt_rank=case[1]) + initializers.lecun_initializer(case[0], tt_rank=case[1], + dtype=self.tf_dtype) + + +class InitializersTestFloat32(tf.test.TestCase, _InitializersTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class InitializersTestFloat64(tf.test.TestCase, _InitializersTest): + np_dtype = np.float64 + tf_dtype = tf.float64 if __name__ == "__main__": From 3e4283f6c4dedee83468f34eb223e1c53943a44a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 19:50:21 +0000 Subject: [PATCH 081/233] bug fix: correct dtype for ones_like and zeros_like --- t3f/initializers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/t3f/initializers.py b/t3f/initializers.py index 7036416c..936a29bd 100644 --- a/t3f/initializers.py +++ b/t3f/initializers.py @@ -437,9 +437,9 @@ def ones_like(tt): else: shape = shapes.lazy_raw_shape(tt) if tt.is_tt_matrix(): - return matrix_ones(shape) + return matrix_ones(shape, dtype=tt.dtype) else: - return tensor_ones(shape[0, :]) + return tensor_ones(shape[0, :], dtype=tt.dtype) def zeros_like(tt): @@ -461,9 +461,9 @@ def zeros_like(tt): else: shape = shapes.lazy_raw_shape(tt) if tt.is_tt_matrix(): - return matrix_zeros(shape) + return matrix_zeros(shape, dtype=tt.dtype) else: - return tensor_zeros(shape[0, :]) + return tensor_zeros(shape[0, :], dtype=tt.dtype) def random_tensor(shape, tt_rank=2, mean=0., stddev=1., dtype=tf.float32): From d940f947087b16a5e3260e085c170b2851f2cc9e Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 19:59:14 +0000 Subject: [PATCH 082/233] kronecker: test with different dtypes --- t3f/kronecker_test.py | 79 +++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/t3f/kronecker_test.py b/t3f/kronecker_test.py index 84a07940..04557c9a 100644 --- a/t3f/kronecker_test.py +++ b/t3f/kronecker_test.py @@ -8,23 +8,26 @@ from t3f import variables from t3f import kronecker as kr -class KroneckerTest(tf.test.TestCase): +class _KroneckerTest(): def testIsKronNonKron(self): # Tests _is_kron on a non-Kronecker matrix - initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=2) + initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=2, + dtype=self.tf_dtype) tt_mat = variables.get_variable('tt_mat', initializer=initializer) self.assertFalse(kr._is_kron(tt_mat)) def testIsKronKron(self): # Tests _is_kron on a Kronecker matrix - initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=1) + initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=1, + dtype=self.tf_dtype) kron_mat = variables.get_variable('kron_mat', initializer=initializer) self.assertTrue(kr._is_kron(kron_mat)) def testDet(self): # Tests the determinant function - initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1) + initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1, + dtype=self.tf_dtype) kron_mat = variables.get_variable('kron_mat', initializer=initializer) init_op = tf.global_variables_initializer() with self.test_session() as sess: @@ -40,11 +43,13 @@ def testSlogDet(self): # the current version is platform-dependent tf.set_random_seed(5) # negative derminant - initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1) + initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1, + dtype=self.tf_dtype) kron_neg = variables.get_variable('kron_neg', initializer=initializer) tf.set_random_seed(1) # positive determinant - initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1) + initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1, + dtype=self.tf_dtype) kron_pos = variables.get_variable('kron_pos', initializer=initializer) init_op = tf.global_variables_initializer() @@ -64,7 +69,8 @@ def testSlogDet(self): def testInv(self): # Tests the inv function - initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1) + initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1, + dtype=self.tf_dtype) kron_mat = variables.get_variable('kron_mat', initializer=initializer) init_op = tf.global_variables_initializer() with self.test_session() as sess: @@ -79,45 +85,50 @@ def testCholesky(self): # generating two symmetric positive-definite tt-cores L_1 = np.tril(np.random.normal(scale=2., size=(2, 2))) + L_1 = L_1.astype(self.np_dtype) L_2 = np.tril(np.random.normal(scale=2., size=(3, 3))) + L_2 = L_2.astype(self.np_dtype) K_1 = L_1.dot(L_1.T) K_2 = L_2.dot(L_2.T) K = np.kron(K_1, K_2) initializer = TensorTrain([K_1[None, :, :, None], - K_2[None, :, :, None]], - tt_ranks=7*[1]) + K_2[None, :, :, None]], + tt_ranks=7*[1]) kron_mat = variables.get_variable('kron_mat', initializer=initializer) init_op = tf.global_variables_initializer() with self.test_session() as sess: sess.run(init_op) desired = np.linalg.cholesky(K) actual = ops.full(kr.cholesky(kron_mat)).eval() - self.assertAllClose(desired, actual) + self.assertAllClose(desired, actual, atol=1e-5, rtol=1e-5) -class BatchKroneckerTest(tf.test.TestCase): +class _BatchKroneckerTest(): def testIsKronNonKron(self): # Tests _is_kron on a non-Kronecker matrix batch initializer = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=2, - batch_size=3) + batch_size=3, + dtype=self.tf_dtype) tt_mat_batch = variables.get_variable('tt_mat_batch', - initializer=initializer) + initializer=initializer) self.assertFalse(kr._is_kron(tt_mat_batch)) def testIsKronKron(self): # Tests _is_kron on a Kronecker matrix batch initializer = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=1, - batch_size=3) + batch_size=3, + dtype=self.tf_dtype) kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) + initializer=initializer) self.assertTrue(kr._is_kron(kron_mat_batch)) def testDet(self): # Tests the determinant function initializer = initializers.random_matrix_batch(((2, 3, 2), (2, 3, 2)), - tt_rank=1, batch_size=3) + tt_rank=1, batch_size=3, + dtype=self.tf_dtype) kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) + initializer=initializer) init_op = tf.global_variables_initializer() with self.test_session() as sess: sess.run(init_op) @@ -130,9 +141,10 @@ def testSlogDet(self): tf.set_random_seed(1) # negative and positive determinants initializer = initializers.random_matrix_batch(((2, 3), (2, 3)), tt_rank=1, - batch_size=3) + batch_size=3, + dtype=self.tf_dtype) kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) + initializer=initializer) init_op = tf.global_variables_initializer() with self.test_session() as sess: @@ -147,9 +159,10 @@ def testSlogDet(self): def testInv(self): # Tests the inv function initializer = initializers.random_matrix_batch(((2, 3, 2), (2, 3, 2)), - tt_rank=1, batch_size=3) + tt_rank=1, batch_size=3, + dtype=self.tf_dtype) kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) + initializer=initializer) init_op = tf.global_variables_initializer() with self.test_session() as sess: sess.run(init_op) @@ -163,13 +176,15 @@ def testCholesky(self): # generating two symmetric positive-definite tt-cores L_1 = np.tril(np.random.normal(scale=2., size=(4, 2, 2))) + L_1 = L_1.astype(self.np_dtype) L_2 = np.tril(np.random.normal(scale=2., size=(4, 3, 3))) + L_2 = L_2.astype(self.np_dtype) K_1 = np.einsum('ijk,ilk->ijl', L_1, L_1) K_2 = np.einsum('ijk,ilk->ijl', L_2, L_2) initializer = TensorTrainBatch([K_1[:, None, :, :, None], K_2[:, None, :, :, None]], tt_ranks=7*[1]) kron_mat_batch = variables.get_variable('kron_mat_batch', - initializer=initializer) + initializer=initializer) init_op = tf.global_variables_initializer() with self.test_session() as sess: sess.run(init_op) @@ -178,5 +193,25 @@ def testCholesky(self): self.assertAllClose(desired, actual) +class KroneckerTestFloat32(tf.test.TestCase, _KroneckerTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class KroneckerTestFloat64(tf.test.TestCase, _KroneckerTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + +class BatchKroneckerTestFloat32(tf.test.TestCase, _BatchKroneckerTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class BatchKroneckerTestFloat64(tf.test.TestCase, _BatchKroneckerTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From 7688923d5068204f89ceebee37eeae081de9c30e Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 20:18:47 +0000 Subject: [PATCH 083/233] riemannian: test different dtypes --- t3f/riemannian_test.py | 111 ++++++++++++++++++++++++++++------------- 1 file changed, 75 insertions(+), 36 deletions(-) diff --git a/t3f/riemannian_test.py b/t3f/riemannian_test.py index 25ddb7d1..b02653c9 100644 --- a/t3f/riemannian_test.py +++ b/t3f/riemannian_test.py @@ -9,11 +9,11 @@ from t3f import batch_ops -class RiemannianTest(tf.test.TestCase): +class _RiemannianTest(): def testProjectOnItself(self): # Projection of X into the tangent space of itself is X: P_x(x) = x. - tens = initializers.random_tensor((2, 3, 4)) + tens = initializers.random_tensor((2, 3, 4), dtype=self.tf_dtype) proj = riemannian.project_sum(tens, tens) with self.test_session() as sess: actual_val, desired_val = sess.run((ops.full(proj), ops.full(tens))) @@ -32,6 +32,8 @@ def testProject(self): [[-0.19279465], [ 0.524976 ], [-0.40149197]]]) + convert = lambda t: np.array(t, dtype=self.np_dtype) + tangent_tens_cores = list([convert(t) for t in tangent_tens_cores]) tangent_tens = TensorTrain(tangent_tens_cores, (4, 3), (1, 2, 1)) tens_cores = ([[[-1.01761142, 0.36075896, -0.2493624 ], [-0.99896565, -1.12685474, 1.02832458], @@ -47,19 +49,25 @@ def testProject(self): [[ 0.76616274], [ 0.6577514 ], [ 2.13703185]]]) - tens = TensorTrain(tens_cores, (4, 3), (1, 3, 1)) + tens_cores = list([convert(t) for t in tens_cores]) + tens = TensorTrain((tens_cores), (4, 3), (1, 3, 1)) desired_projection = [[-0.67638254, -1.17163914, 0.29850939], [-1.66479093, -0.99003251, 2.46629195], [-0.04847773, -0.72908174, 0.20142675], [ 0.34431125, -0.20935516, -1.15864246]] proj = riemannian.project_sum(tens, tangent_tens) + proj_full = ops.full(proj) with self.test_session() as sess: - self.assertAllClose(desired_projection, ops.full(proj).eval()) + proj_v = proj_full.eval() + self.assertAllClose(desired_projection, proj_v) + self.assertEqual(self.np_dtype, proj_v.dtype) def testProjectSum(self): # Test projecting a batch of TT-tensors. - tens = initializers.random_tensor_batch((2, 3, 4), batch_size=3) - tangent_tens = initializers.random_tensor((2, 3, 4), 3) + tens = initializers.random_tensor_batch((2, 3, 4), batch_size=3, + dtype=self.tf_dtype) + tangent_tens = initializers.random_tensor((2, 3, 4), 3, + dtype=self.tf_dtype) weighted_sum = tens[0] + tens[1] + tens[2] direct_proj = riemannian.project_sum(weighted_sum, tangent_tens) actual_proj = riemannian.project_sum(tens, tangent_tens) @@ -70,9 +78,11 @@ def testProjectSum(self): def testProjectWeightedSum(self): # Test projecting a batch of TT-tensors with providing coefs. - tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4) + tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4, + dtype=self.tf_dtype) coef = [0.1, -2, 0, 0.4] - tangent_tens = initializers.random_tensor((2, 3, 4), 4) + tangent_tens = initializers.random_tensor((2, 3, 4), 4, + dtype=self.tf_dtype) weighted_sum = coef[0] * tens[0] + coef[1] * tens[1] + coef[2] * tens[2] weighted_sum += coef[3] * tens[3] direct_proj = riemannian.project_sum(weighted_sum, tangent_tens) @@ -85,10 +95,12 @@ def testProjectWeightedSum(self): def testProjectWeightedSumMultipleOutputs(self): # Test projecting a batch of TT-tensors with providing weights and outputing # several TT objects with different weights. - tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4) + tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4, + dtype=self.tf_dtype) np.random.seed(0) - weights = np.random.randn(4, 2).astype(np.float32) - tangent_tens = initializers.random_tensor((2, 3, 4), 4) + weights = np.random.randn(4, 2).astype(self.np_dtype) + tangent_tens = initializers.random_tensor((2, 3, 4), 4, + dtype=self.tf_dtype) weighted_sum_1 = weights[0, 0] * tens[0] + weights[1, 0] * tens[1] +\ weights[2, 0] * tens[2] + weights[3, 0] * tens[3] weighted_sum_2 = weights[0, 1] * tens[0] + weights[1, 1] * tens[1] +\ @@ -107,7 +119,8 @@ def testProjectWeightedSumMultipleOutputs(self): def testProjectMatrixOnItself(self): # Project a TT-matrix on itself. # Projection of X into the tangent space of itself is X: P_x(x) = x. - tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4))) + tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4)), + dtype=self.tf_dtype) proj = riemannian.project_sum(tt_mat, tt_mat) with self.test_session() as sess: actual_val, desired_val = sess.run((ops.full(proj), ops.full(tt_mat))) @@ -115,8 +128,10 @@ def testProjectMatrixOnItself(self): def testCompareProjectSumAndProject(self): # Compare results of project_sum and project. - tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4) - tangent_tens = initializers.random_tensor((2, 3, 4), 4) + tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4, + dtype=self.tf_dtype) + tangent_tens = initializers.random_tensor((2, 3, 4), 4, + dtype=self.tf_dtype) project_sum = riemannian.project_sum(tens, tangent_tens, tf.eye(4)) project = riemannian.project(tens, tangent_tens) with self.test_session() as sess: @@ -126,10 +141,13 @@ def testCompareProjectSumAndProject(self): def testProjectMatmul(self): # Project a TT-matrix times TT-vector on a TT-vector. - tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4))) + tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4)), + dtype=self.tf_dtype) tt_vec_what = initializers.random_matrix_batch(((2, 3, 4), None), - batch_size=3) - tt_vec_where = initializers.random_matrix(((2, 3, 4), None)) + batch_size=3, + dtype=self.tf_dtype) + tt_vec_where = initializers.random_matrix(((2, 3, 4), None), + dtype=self.tf_dtype) proj = riemannian.project_matmul(tt_vec_what, tt_vec_where, tt_mat) matvec = ops.matmul(tt_mat, tt_vec_what) proj_desired = riemannian.project(matvec, tt_vec_where) @@ -139,9 +157,11 @@ def testProjectMatmul(self): def testPairwiseFlatInnerTensor(self): # Compare pairwise_flat_inner_projected against naive implementation. - what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3) - what2 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=4) - where = initializers.random_tensor((2, 3, 4), 3) + what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3, + dtype=self.tf_dtype) + what2 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=4, + dtype=self.tf_dtype) + where = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) desired = batch_ops.pairwise_flat_inner(projected1, projected2) @@ -153,7 +173,7 @@ def testPairwiseFlatInnerTensor(self): with self.assertRaises(ValueError): # Second argument is not a projection on the tangent space. riemannian.pairwise_flat_inner_projected(projected1, what2) - where2 = initializers.random_tensor((2, 3, 4), 3) + where2 = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) another_projected2 = riemannian.project(what2, where2) with self.assertRaises(ValueError): # The arguments are projections on different tangent spaces. @@ -161,9 +181,12 @@ def testPairwiseFlatInnerTensor(self): def testPairwiseFlatInnerMatrix(self): # Compare pairwise_flat_inner_projected against naive implementation. - what1 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=3) - what2 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=4) - where = initializers.random_matrix(((2, 3, 4), None), 3) + what1 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=3, + dtype=self.tf_dtype) + what2 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=4, + dtype=self.tf_dtype) + where = initializers.random_matrix(((2, 3, 4), None), 3, + dtype=self.tf_dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) desired = batch_ops.pairwise_flat_inner(projected1, projected2) @@ -175,7 +198,8 @@ def testPairwiseFlatInnerMatrix(self): with self.assertRaises(ValueError): # Second argument is not a projection on the tangent space. riemannian.pairwise_flat_inner_projected(projected1, what2) - where2 = initializers.random_matrix(((2, 3, 4), None), 3) + where2 = initializers.random_matrix(((2, 3, 4), None), 3, + dtype=self.tf_dtype) another_projected2 = riemannian.project(what2, where2) with self.assertRaises(ValueError): # The arguments are projections on different tangent spaces. @@ -183,9 +207,11 @@ def testPairwiseFlatInnerMatrix(self): def testAddNProjected(self): # Add several TT-objects from the same tangent space. - what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3) - what2 = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=3) - where = initializers.random_tensor((2, 3, 4), 3) + what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3, + dtype=self.tf_dtype) + what2 = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=3, + dtype=self.tf_dtype) + where = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) desired = ops.full(projected1 + projected2) @@ -197,7 +223,7 @@ def testAddNProjected(self): with self.assertRaises(ValueError): # Second argument is not a projection on the tangent space. riemannian.add_n_projected((projected1, what2)) - where2 = initializers.random_tensor((2, 3, 4), 3) + where2 = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) another_projected2 = riemannian.project(what2, where2) with self.assertRaises(ValueError): # The arguments are projections on different tangent spaces. @@ -205,9 +231,9 @@ def testAddNProjected(self): def testWeightedAddNProjected(self): # Add several TT-objects from the same tangent space with coefs. - what1 = initializers.random_tensor((2, 3, 4), 4) - what2 = initializers.random_tensor((2, 3, 4), 1) - where = initializers.random_tensor((2, 3, 4), 3) + what1 = initializers.random_tensor((2, 3, 4), 4, dtype=self.tf_dtype) + what2 = initializers.random_tensor((2, 3, 4), 1, dtype=self.tf_dtype) + where = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) desired = ops.full(1.2 * projected1 + -2.0 * projected2) @@ -220,7 +246,7 @@ def testWeightedAddNProjected(self): with self.assertRaises(ValueError): # Second argument is not a projection on the tangent space. riemannian.add_n_projected((projected1, what2), coef=[1.2, -2.0]) - where2 = initializers.random_tensor((2, 3, 4), 3) + where2 = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) another_projected2 = riemannian.project(what2, where2) with self.assertRaises(ValueError): # The arguments are projections on different tangent spaces. @@ -229,9 +255,11 @@ def testWeightedAddNProjected(self): def testWeightedAddNProjectedBatch(self): # Add several TT-batches from the same tangent space with coefs. - what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3) - what2 = initializers.random_tensor_batch((2, 3, 4), 1, batch_size=3) - where = initializers.random_tensor((2, 3, 4), 3) + what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3, + dtype=self.tf_dtype) + what2 = initializers.random_tensor_batch((2, 3, 4), 1, batch_size=3, + dtype=self.tf_dtype) + where = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) @@ -246,5 +274,16 @@ def testWeightedAddNProjectedBatch(self): desired_val, actual_val = sess.run((desired, actual)) self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) + +class RiemannianTestFloat32(tf.test.TestCase, _RiemannianTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class RiemannianTestFloat64(tf.test.TestCase, _RiemannianTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From cd3e85bac0cf35d527ceaaaf7e51fd4db777d54d Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 20:19:11 +0000 Subject: [PATCH 084/233] fix bug: convert coefs into appropriate dtype --- t3f/riemannian.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/t3f/riemannian.py b/t3f/riemannian.py index 21a49748..667d0d99 100644 --- a/t3f/riemannian.py +++ b/t3f/riemannian.py @@ -42,6 +42,7 @@ def project_sum(what, where, weights=None): if weights is not None: weights = tf.convert_to_tensor(weights) + weights = tf.cast(weights, where.dtype) if not isinstance(where, TensorTrain): raise ValueError('The first argument should be a TensorTrain object, got ' @@ -662,6 +663,7 @@ def add_n_projected(tt_objects, coef=None): projection_on)) if coef is not None: coef = tf.convert_to_tensor(coef) + coef = tf.cast(coef, dtype=tt_objects[0].dtype) if coef.get_shape().ndims > 1: # In batch case we will need to multiply each core by this coefficients # along the first axis. To do it need to reshape the coefs to match From 57d00ae0f47b019e01a5135006f34a585abfda2f Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 20:34:48 +0000 Subject: [PATCH 085/233] variables: test dtypes --- t3f/variables_test.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/t3f/variables_test.py b/t3f/variables_test.py index de96a22d..3d7c2d6c 100644 --- a/t3f/variables_test.py +++ b/t3f/variables_test.py @@ -5,10 +5,11 @@ from t3f import ops from t3f import initializers -class VariablesTest(tf.test.TestCase): + +class _VariablesTest(): def testGetExistingVariable(self): - init = initializers.random_tensor([2, 3, 2], tt_rank=2) + init = initializers.random_tensor([2, 3, 2], tt_rank=2, dtype=self.tf_dtype) tt_1 = variables.get_variable('tt_1', initializer=init) with tf.variable_scope('test'): tt_2 = variables.get_variable('tt_2', initializer=init) @@ -23,13 +24,14 @@ def testGetExistingVariable(self): variables.get_variable('tt_3') with tf.variable_scope('', reuse=True): - tt_1_copy = variables.get_variable('tt_1') + tt_1_copy = variables.get_variable('tt_1', dtype=self.tf_dtype) self.assertAllClose(ops.full(tt_1).eval(), ops.full(tt_1_copy).eval()) with tf.variable_scope('', reuse=True): # Again try to retrieve an existing variable, but pass an initializer # and check that it still works. - tt_1_copy = variables.get_variable('tt_1', initializer=0 * init) + tt_1_copy = variables.get_variable('tt_1', initializer=0 * init, + dtype=self.tf_dtype) self.assertAllClose(ops.full(tt_1).eval(), ops.full(tt_1_copy).eval()) with self.assertRaises(ValueError): @@ -43,15 +45,16 @@ def testGetExistingVariable(self): variables.get_variable('tt_2') with tf.variable_scope('test', reuse=True): - tt_2_copy = variables.get_variable('tt_2') + tt_2_copy = variables.get_variable('tt_2', dtype=self.tf_dtype) self.assertAllClose(ops.full(tt_2).eval(), ops.full(tt_2_copy).eval()) def testAttributes(self): # Test that after converting an initializer into a variable all the # attributes stays the same. - tens = initializers.random_tensor([2, 3, 2], tt_rank=2) + tens = initializers.random_tensor([2, 3, 2], tt_rank=2, dtype=self.tf_dtype) tens_v = variables.get_variable('tt_tens', initializer=tens) - mat = initializers.random_matrix([[3, 2, 2], [3, 3, 3]], tt_rank=3) + mat = initializers.random_matrix([[3, 2, 2], [3, 3, 3]], tt_rank=3, + dtype=self.tf_dtype) mat_v = variables.get_variable('tt_mat', initializer=mat) for (init, var) in [[tens, tens_v], [mat, mat_v]]: self.assertEqual(init.get_shape(), var.get_shape()) @@ -61,9 +64,11 @@ def testAttributes(self): self.assertEqual(init.is_tt_matrix(), var.is_tt_matrix()) def testAssign(self): - old_init = initializers.random_tensor([2, 3, 2], tt_rank=2) + old_init = initializers.random_tensor([2, 3, 2], tt_rank=2, + dtype=self.tf_dtype) tt = variables.get_variable('tt', initializer=old_init) - new_init = initializers.random_tensor([2, 3, 2], tt_rank=2) + new_init = initializers.random_tensor([2, 3, 2], tt_rank=2, + dtype=self.tf_dtype) assigner = variables.assign(tt, new_init) with self.test_session(): tf.global_variables_initializer().run() @@ -78,5 +83,15 @@ def testAssign(self): self.assertGreater(rel_diff, 0.2) +class VariablesTestFloat32(tf.test.TestCase, _VariablesTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class VariablesTestFloat64(tf.test.TestCase, _VariablesTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From a8da2cb3ca5fae1698e17ab676fd0f8b4093b061 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 20:35:27 +0000 Subject: [PATCH 086/233] bug fix: sharing variables of dtype != tf.float32 --- t3f/variables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/variables.py b/t3f/variables.py index 7e2422a5..fd5c8799 100644 --- a/t3f/variables.py +++ b/t3f/variables.py @@ -76,7 +76,7 @@ def get_variable(name, with tf.variable_scope(name): # Try to get the first core through tf.get_variable to check that we don't # violate reuse: it will raise a ValueError otherwise. - tf.get_variable('core_0') + tf.get_variable('core_0', dtype=dtype) return found_v else: # Create new variable. From d4343cba2a4b834efd305ffe4b2a1978be53fdbb Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 20:55:07 +0000 Subject: [PATCH 087/233] test dtypes of tt classes --- t3f/tensor_train_batch_test.py | 22 +++++++++++++++++---- t3f/tensor_train_test.py | 35 +++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/t3f/tensor_train_batch_test.py b/t3f/tensor_train_batch_test.py index 41894f1e..98f8cdb3 100644 --- a/t3f/tensor_train_batch_test.py +++ b/t3f/tensor_train_batch_test.py @@ -1,13 +1,15 @@ +import numpy as np import tensorflow as tf from t3f import initializers from t3f import ops -class TensorTrainBatchTest(tf.test.TestCase): +class _TensorTrainBatchTest(): def testTensorIndexing(self): - tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3) + tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3, + dtype=self.tf_dtype) with self.test_session() as sess: desired = ops.full(tens)[:, :, :, :] actual = ops.full(tens[:, :, :, :]) @@ -47,7 +49,8 @@ def testTensorIndexing(self): tens[1, 1] def testPlaceholderTensorIndexing(self): - tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3) + tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3, + dtype=self.tf_dtype) with self.test_session() as sess: start = tf.placeholder(tf.int32) end = tf.placeholder(tf.int32) @@ -74,10 +77,21 @@ def testPlaceholderTensorIndexing(self): def testShapeOverflow(self): large_shape = [10] * 20 - tensor = initializers.random_matrix_batch([large_shape, large_shape], batch_size=5) + tensor = initializers.random_matrix_batch([large_shape, large_shape], + batch_size=5, dtype=self.tf_dtype) shape = tensor.get_shape() self.assertEqual([5, 10 ** 20, 10 ** 20], shape) +class TensorTrainBatchTestFloat32(tf.test.TestCase, _TensorTrainBatchTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class TensorTrainBatchTestFloat64(tf.test.TestCase, _TensorTrainBatchTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() diff --git a/t3f/tensor_train_test.py b/t3f/tensor_train_test.py index ec99dd7d..d22f9c93 100644 --- a/t3f/tensor_train_test.py +++ b/t3f/tensor_train_test.py @@ -1,3 +1,4 @@ +import numpy as np import tensorflow as tf from t3f import tensor_train @@ -5,7 +6,7 @@ from t3f import ops -class TensorTrainTest(tf.test.TestCase): +class _TensorTrainTest(): def testValidateTTCores2d(self): schedule = (((1, 1, 1, 1), (1, 1, 1), True), @@ -26,8 +27,8 @@ def testValidateTTCores2d(self): ((1, 2, 1, 1), (1, 2, 1), False)) for tt_ranks, claimed_tt_ranks, desired in schedule: - a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1])) - b = tf.random_normal((tt_ranks[2], 9, tt_ranks[3])) + a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1]), dtype=self.tf_dtype) + b = tf.random_normal((tt_ranks[2], 9, tt_ranks[3]), dtype=self.tf_dtype) with self.test_session(): actual = tensor_train._are_tt_cores_valid((a, b), (10, 9), claimed_tt_ranks) @@ -41,7 +42,7 @@ def testValidateTTCores2d(self): tensor_train.TensorTrain((a, b), (10, 9), claimed_tt_ranks) # Make dtypes inconsistent. - b_new = tf.cast(b, tf.float64) + b_new = tf.cast(b, tf.float16) actual = tensor_train._are_tt_cores_valid((a, b_new), (10, 9), claimed_tt_ranks) self.assertEqual(False, actual) @@ -71,9 +72,9 @@ def testValidateTTCores3d(self): ((1, 2, 2, 3, 3, 1), None, True)) for tt_ranks, claimed_tt_ranks, desired in schedule: - a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1])) - b = tf.random_normal((tt_ranks[2], 1, tt_ranks[3])) - c = tf.random_normal((tt_ranks[4], 2, tt_ranks[5])) + a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1]), dtype=self.tf_dtype) + b = tf.random_normal((tt_ranks[2], 1, tt_ranks[3]), dtype=self.tf_dtype) + c = tf.random_normal((tt_ranks[4], 2, tt_ranks[5]), dtype=self.tf_dtype) with self.test_session(): actual = tensor_train._are_tt_cores_valid((a, b, c), (10, 1, 2), claimed_tt_ranks) @@ -87,7 +88,7 @@ def testValidateTTCores3d(self): tensor_train.TensorTrain((a, b, c), (10, 1, 2), claimed_tt_ranks) # Make dtypes inconsistent. - b_new = tf.cast(b, tf.float64) + b_new = tf.cast(b, tf.float16) actual = tensor_train._are_tt_cores_valid((a, b_new, c), (10, 1, 2), claimed_tt_ranks) self.assertEqual(False, actual) @@ -95,7 +96,7 @@ def testValidateTTCores3d(self): tensor_train.TensorTrain((a, b_new, c), (10, 1, 2), claimed_tt_ranks) def testTensorIndexing(self): - tens = initializers.random_tensor((3, 3, 4)) + tens = initializers.random_tensor((3, 3, 4), dtype=self.tf_dtype) with self.test_session() as sess: desired = ops.full(tens)[:, :, :] actual = ops.full(tens[:, :, :]) @@ -125,7 +126,7 @@ def testTensorIndexing(self): tens[1, 1] def testPlaceholderTensorIndexing(self): - tens = initializers.random_tensor((3, 3, 4)) + tens = initializers.random_tensor((3, 3, 4), dtype=self.tf_dtype) with self.test_session() as sess: start = tf.placeholder(tf.int32) end = tf.placeholder(tf.int32) @@ -136,9 +137,21 @@ def testPlaceholderTensorIndexing(self): def testShapeOverflow(self): large_shape = [10] * 20 - matrix = initializers.matrix_zeros([large_shape, large_shape]) + matrix = initializers.matrix_zeros([large_shape, large_shape], + dtype=self.tf_dtype) shape = matrix.get_shape() self.assertEqual([10 ** 20, 10 ** 20], shape) + +class TensorTrainTestFloat32(tf.test.TestCase, _TensorTrainTest): + np_dtype = np.float32 + tf_dtype = tf.float32 + + +class TensorTrainTestFloat64(tf.test.TestCase, _TensorTrainTest): + np_dtype = np.float64 + tf_dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From 16e8ca68640cf04d9a59cc5845865145126aa3a4 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 28 Oct 2018 20:56:56 +0000 Subject: [PATCH 088/233] Since now every test is run twice for two different dtypes, reduce the number of tf versions tested against so that testing is not too slow --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 45f84193..e075c33c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,6 @@ python: env: matrix: - TF_VERSION=1.0 - - TF_VERSION=1.7 - - TF_VERSION=1.8 - - TF_VERSION=1.9 - TF_VERSION=1.10 - TF_VERSION=1.11 # command to install dependencies From 17e153403ef7bdd4557ad55e4eedd1c3f9350d2e Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 29 Oct 2018 20:07:02 +0000 Subject: [PATCH 089/233] let things like tf.ones((), dtype=tf.float64) * tt_object raise dtype error like it usually happens with TF --- t3f/approximate.py | 3 +-- t3f/batch_ops.py | 3 +-- t3f/regularizers.py | 8 ++------ t3f/riemannian.py | 6 ++---- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/t3f/approximate.py b/t3f/approximate.py index 3c560ee6..259a10f7 100644 --- a/t3f/approximate.py +++ b/t3f/approximate.py @@ -72,8 +72,7 @@ def reduce_sum_batch(tt_batch, max_tt_rank, coef=None): is_batch_output = False if coef is not None: - coef = tf.convert_to_tensor(coef) - coef = tf.cast(coef, tt_batch.dtype) + coef = tf.convert_to_tensor(coef, dtype=tt_batch.dtype) if len(coef.get_shape()) == 1: tt_batch = batch_ops.multiply_along_batch_dim(tt_batch, coef) elif len(coef.get_shape()) == 2: diff --git a/t3f/batch_ops.py b/t3f/batch_ops.py index d1cf7ebf..dc03a430 100644 --- a/t3f/batch_ops.py +++ b/t3f/batch_ops.py @@ -60,8 +60,7 @@ def multiply_along_batch_dim(batch_tt, weights): Returns: TensorTrainBatch """ - weights = tf.convert_to_tensor(weights) - weights = tf.cast(weights, batch_tt.dtype) + weights = tf.convert_to_tensor(weights, dtype=batch_tt.dtype) tt_cores = list(batch_tt.tt_cores) if batch_tt.is_tt_matrix(): weights = weights[:, tf.newaxis, tf.newaxis, tf.newaxis, tf.newaxis] diff --git a/t3f/regularizers.py b/t3f/regularizers.py index 26b8cf67..764d2f74 100644 --- a/t3f/regularizers.py +++ b/t3f/regularizers.py @@ -31,9 +31,7 @@ def l2_regularizer(scale, scope=None): def l2(tt): """Applies l2 regularization to TensorTrain object.""" with tf.name_scope(scope, 'l2_regularizer', [tt]) as name: - my_scale = tf.convert_to_tensor(scale, - dtype=tt.dtype.base_dtype, - name='scale') + my_scale = tf.convert_to_tensor(scale, dtype=tt.dtype, name='scale') return tf.multiply(my_scale, ops.frobenius_norm_squared(tt), name=name) return l2 @@ -68,9 +66,7 @@ def cores_regularizer(core_regularizer, scale, scope=None): def regularizer(tt): """Applies the regularization to TensorTrain object.""" with tf.name_scope(scope, 'l2_regularizer', [tt]) as name: - my_scale = tf.convert_to_tensor(scale, - dtype=tt.dtype.base_dtype, - name='scale') + my_scale = tf.convert_to_tensor(scale, dtype=tt.dtype, name='scale') penalty = 0.0 for i in range(tt.ndims()): penalty += core_regularizer(tt.tt_cores[i]) diff --git a/t3f/riemannian.py b/t3f/riemannian.py index 667d0d99..a3faf434 100644 --- a/t3f/riemannian.py +++ b/t3f/riemannian.py @@ -41,8 +41,7 @@ def project_sum(what, where, weights=None): what = shapes.expand_batch_dim(what) if weights is not None: - weights = tf.convert_to_tensor(weights) - weights = tf.cast(weights, where.dtype) + weights = tf.convert_to_tensor(weights, dtype=where.dtype) if not isinstance(where, TensorTrain): raise ValueError('The first argument should be a TensorTrain object, got ' @@ -662,8 +661,7 @@ def add_n_projected(tt_objects, coef=None): 'least the pointers are different.' % (tt.projection_on, projection_on)) if coef is not None: - coef = tf.convert_to_tensor(coef) - coef = tf.cast(coef, dtype=tt_objects[0].dtype) + coef = tf.convert_to_tensor(coef, dtype=tt_objects[0].dtype) if coef.get_shape().ndims > 1: # In batch case we will need to multiply each core by this coefficients # along the first axis. To do it need to reshape the coefs to match From 73ff1fb2733b27b37675e023fb70d266a92a8357 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 29 Oct 2018 20:07:34 +0000 Subject: [PATCH 090/233] fix test: pass np array instead of tf array to make sure the dtype doesnt matter --- t3f/riemannian_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/riemannian_test.py b/t3f/riemannian_test.py index b02653c9..dd1e3901 100644 --- a/t3f/riemannian_test.py +++ b/t3f/riemannian_test.py @@ -132,7 +132,7 @@ def testCompareProjectSumAndProject(self): dtype=self.tf_dtype) tangent_tens = initializers.random_tensor((2, 3, 4), 4, dtype=self.tf_dtype) - project_sum = riemannian.project_sum(tens, tangent_tens, tf.eye(4)) + project_sum = riemannian.project_sum(tens, tangent_tens, np.eye(4)) project = riemannian.project(tens, tangent_tens) with self.test_session() as sess: res = sess.run((ops.full(project_sum), ops.full(project))) From 39a5de5b2db38edec35c974b025e820903387bd6 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 29 Oct 2018 20:18:30 +0000 Subject: [PATCH 091/233] remove code duplication of np and tf dtype --- t3f/approximate_test.py | 4 +-- t3f/batch_ops_test.py | 2 -- t3f/decompositions_test.py | 18 ++++------- t3f/initializers_test.py | 14 ++++---- t3f/kronecker_test.py | 12 +++---- t3f/ops_test.py | 59 +++++++++++++++------------------- t3f/riemannian_test.py | 8 ++--- t3f/tensor_train_batch_test.py | 2 -- t3f/tensor_train_test.py | 2 -- t3f/variables_test.py | 2 -- 10 files changed, 47 insertions(+), 76 deletions(-) diff --git a/t3f/approximate_test.py b/t3f/approximate_test.py index 53e492db..41d8e06c 100644 --- a/t3f/approximate_test.py +++ b/t3f/approximate_test.py @@ -90,7 +90,7 @@ def desired(tt_batch, coef): coef = [[1., 0.1], [0.9, -0.2], [0.3, 0.3]] - coef = np.array(coef).astype(self.np_dtype) + coef = np.array(coef).astype(self.tf_dtype.as_numpy_dtype) res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 6, coef)) res_desired_1 = ops.full(desired(tt_batch, coef[:, 0])) @@ -101,12 +101,10 @@ def desired(tt_batch, coef): class ApproximateTestFloat32(tf.test.TestCase, _ApproximateTest): - np_dtype = np.float32 tf_dtype = tf.float32 class ApproximateTestFloat64(tf.test.TestCase, _ApproximateTest): - np_dtype = np.float64 tf_dtype = tf.float64 diff --git a/t3f/batch_ops_test.py b/t3f/batch_ops_test.py index 61651019..8abc9dcc 100644 --- a/t3f/batch_ops_test.py +++ b/t3f/batch_ops_test.py @@ -174,12 +174,10 @@ def testPairwiseFlatInnerVectorsWithMatrix(self): class BatchOpsTestFloat32(tf.test.TestCase, _BatchOpsTest): - np_dtype = np.float32 tf_dtype = tf.float32 class BatchOpsTestFloat64(tf.test.TestCase, _BatchOpsTest): - np_dtype = np.float64 tf_dtype = tf.float64 diff --git a/t3f/decompositions_test.py b/t3f/decompositions_test.py index 3029198a..7b2ce807 100644 --- a/t3f/decompositions_test.py +++ b/t3f/decompositions_test.py @@ -12,7 +12,7 @@ class _DecompositionsTest(): def testTTTensor(self): shape = (2, 1, 4, 3) np.random.seed(1) - tens = np.random.rand(*shape).astype(self.np_dtype) + tens = np.random.rand(*shape).astype(self.tf_dtype.as_numpy_dtype) tf_tens = tf.constant(tens) tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=3) with self.test_session(): @@ -33,8 +33,8 @@ def testTTTensorSimple(self): # Test that a tensor of ones and of zeros can be converted into TT with # TT-rank 1. shape = (2, 1, 4, 3) - tens_arr = (np.zeros(shape).astype(self.np_dtype), - np.ones(shape).astype(self.np_dtype)) + tens_arr = (np.zeros(shape).astype(self.tf_dtype.as_numpy_dtype), + np.ones(shape).astype(self.tf_dtype.as_numpy_dtype)) for tens in tens_arr: tf_tens = tf.constant(tens) tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=1) @@ -56,7 +56,7 @@ def testTTVector(self): vec_shape = (2, 1, 4, 3) np.random.seed(1) rows = np.prod(vec_shape) - vec = np.random.rand(rows, 1).astype(self.np_dtype) + vec = np.random.rand(rows, 1).astype(self.tf_dtype.as_numpy_dtype) tf_vec = tf.constant(vec) tt_vec = decompositions.to_tt_matrix(tf_vec, (vec_shape, None)) with self.test_session(): @@ -66,7 +66,7 @@ def testTTCompositeRankTensor(self): # Test if a composite rank (list of ranks) can be used for decomposition # for tensor. np.random.seed(1) - np_tensor = np.random.rand(2, 3, 3, 1).astype(self.np_dtype) + np_tensor = np.random.rand(2, 3, 3, 1).astype(self.tf_dtype.as_numpy_dtype) tf_tensor = tf.constant(np_tensor) tt_ranks = [1, 2, 3, 3, 1] @@ -81,7 +81,7 @@ def testTTCompositeRankMatrix(self): out_shape = (1, 2, 2, 1) np.random.seed(1) mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) - mat = mat.astype(self.np_dtype) + mat = mat.astype(self.tf_dtype.as_numpy_dtype) tf_mat = tf.constant(mat) tt_ranks = [10, 20, 30, 40, 30] tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), @@ -96,7 +96,7 @@ def testTTMatrix(self): out_shape = (3, 3, 2, 3) np.random.seed(1) mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) - mat = mat.astype(self.np_dtype) + mat = mat.astype(self.tf_dtype.as_numpy_dtype) tf_mat = tf.constant(mat) tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), max_tt_rank=90) @@ -204,22 +204,18 @@ def testRoundTensor(self): class DecompositionsTestFloat32(tf.test.TestCase, _DecompositionsTest): - np_dtype = np.float32 tf_dtype = tf.float32 class DecompositionsTestFloat64(tf.test.TestCase, _DecompositionsTest): - np_dtype = np.float64 tf_dtype = tf.float64 class DecompositionsBatchTestFloat32(tf.test.TestCase, _DecompositionsBatchTest): - np_dtype = np.float32 tf_dtype = tf.float32 class DecompositionsBatchTestFloat64(tf.test.TestCase, _DecompositionsBatchTest): - np_dtype = np.float64 tf_dtype = tf.float64 diff --git a/t3f/initializers_test.py b/t3f/initializers_test.py index 23f69cd5..8b30ca94 100644 --- a/t3f/initializers_test.py +++ b/t3f/initializers_test.py @@ -11,8 +11,8 @@ def testTensorOnesAndZeros(self): tt_ones = initializers.tensor_ones([2, 3, 4], dtype=self.tf_dtype) tt_zeros = initializers.tensor_zeros([2, 3, 4], dtype=self.tf_dtype) - ones_desired = np.ones((2, 3, 4), dtype=self.np_dtype) - zeros_desired = np.zeros((2, 3, 4), dtype=self.np_dtype) + ones_desired = np.ones((2, 3, 4), dtype=self.tf_dtype.as_numpy_dtype) + zeros_desired = np.zeros((2, 3, 4), dtype=self.tf_dtype.as_numpy_dtype) with self.test_session() as sess: tt_ones_full = sess.run(ops.full(tt_ones)) tt_zeros_full = sess.run(ops.full(tt_zeros)) @@ -33,8 +33,8 @@ def testMatrixOnesAndZeros(self): tt_zeros = initializers.matrix_zeros([[2, 3, 4], [1, 2, 5]], dtype=self.tf_dtype) - ones_desired = np.ones((24, 10), dtype=self.np_dtype) - zeros_desired = np.zeros((24, 10), dtype=self.np_dtype) + ones_desired = np.ones((24, 10), dtype=self.tf_dtype.as_numpy_dtype) + zeros_desired = np.zeros((24, 10), dtype=self.tf_dtype.as_numpy_dtype) bad_shapes = [[[-1, 2, 3], [3, 4, 6]], [[1.5, 2, 4], [2, 5, 6]], [[1], [2, 3]], [2, 3, 4]] @@ -70,9 +70,9 @@ def testOnesLikeAndZerosLike(self): with self.test_session() as sess: bf, cf = sess.run(var_list) self.assertAllClose(bf, np.ones((2, 3, 4))) - self.assertEqual(self.np_dtype, bf.dtype) + self.assertEqual(self.tf_dtype.as_numpy_dtype, bf.dtype) self.assertAllClose(cf, np.zeros((2, 3, 4))) - self.assertEqual(self.np_dtype, cf.dtype) + self.assertEqual(self.tf_dtype.as_numpy_dtype, cf.dtype) with self.assertRaises(ValueError): initializers.ones_like(1) with self.assertRaises(ValueError): @@ -182,12 +182,10 @@ def testInvalidShapeOrRanksLecun(self): class InitializersTestFloat32(tf.test.TestCase, _InitializersTest): - np_dtype = np.float32 tf_dtype = tf.float32 class InitializersTestFloat64(tf.test.TestCase, _InitializersTest): - np_dtype = np.float64 tf_dtype = tf.float64 diff --git a/t3f/kronecker_test.py b/t3f/kronecker_test.py index 04557c9a..ebc1d257 100644 --- a/t3f/kronecker_test.py +++ b/t3f/kronecker_test.py @@ -85,9 +85,9 @@ def testCholesky(self): # generating two symmetric positive-definite tt-cores L_1 = np.tril(np.random.normal(scale=2., size=(2, 2))) - L_1 = L_1.astype(self.np_dtype) + L_1 = L_1.astype(self.tf_dtype.as_numpy_dtype) L_2 = np.tril(np.random.normal(scale=2., size=(3, 3))) - L_2 = L_2.astype(self.np_dtype) + L_2 = L_2.astype(self.tf_dtype.as_numpy_dtype) K_1 = L_1.dot(L_1.T) K_2 = L_2.dot(L_2.T) K = np.kron(K_1, K_2) @@ -176,9 +176,9 @@ def testCholesky(self): # generating two symmetric positive-definite tt-cores L_1 = np.tril(np.random.normal(scale=2., size=(4, 2, 2))) - L_1 = L_1.astype(self.np_dtype) + L_1 = L_1.astype(self.tf_dtype.as_numpy_dtype) L_2 = np.tril(np.random.normal(scale=2., size=(4, 3, 3))) - L_2 = L_2.astype(self.np_dtype) + L_2 = L_2.astype(self.tf_dtype.as_numpy_dtype) K_1 = np.einsum('ijk,ilk->ijl', L_1, L_1) K_2 = np.einsum('ijk,ilk->ijl', L_2, L_2) initializer = TensorTrainBatch([K_1[:, None, :, :, None], @@ -194,22 +194,18 @@ def testCholesky(self): class KroneckerTestFloat32(tf.test.TestCase, _KroneckerTest): - np_dtype = np.float32 tf_dtype = tf.float32 class KroneckerTestFloat64(tf.test.TestCase, _KroneckerTest): - np_dtype = np.float64 tf_dtype = tf.float64 class BatchKroneckerTestFloat32(tf.test.TestCase, _BatchKroneckerTest): - np_dtype = np.float32 tf_dtype = tf.float32 class BatchKroneckerTestFloat64(tf.test.TestCase, _BatchKroneckerTest): - np_dtype = np.float64 tf_dtype = tf.float64 diff --git a/t3f/ops_test.py b/t3f/ops_test.py index e8891311..775fd3ce 100644 --- a/t3f/ops_test.py +++ b/t3f/ops_test.py @@ -13,8 +13,8 @@ class _TTTensorTest(): def testFullTensor2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(10, rank).astype(self.np_dtype) - b = np.random.rand(rank, 9).astype(self.np_dtype) + a = np.random.rand(10, rank).astype(self.tf_dtype.as_numpy_dtype) + b = np.random.rand(rank, 9).astype(self.tf_dtype.as_numpy_dtype) tt_cores = (a.reshape(1, 10, rank), b.reshape(rank, 9, 1)) desired = np.dot(a, b) with self.test_session(): @@ -25,9 +25,9 @@ def testFullTensor2d(self): def testFullTensor3d(self): np.random.seed(1) for rank_1 in [1, 2]: - a = np.random.rand(10, rank_1).astype(self.np_dtype) - b = np.random.rand(rank_1, 9, 3).astype(self.np_dtype) - c = np.random.rand(3, 8).astype(self.np_dtype) + a = np.random.rand(10, rank_1).astype(self.tf_dtype.as_numpy_dtype) + b = np.random.rand(rank_1, 9, 3).astype(self.tf_dtype.as_numpy_dtype) + c = np.random.rand(3, 8).astype(self.tf_dtype.as_numpy_dtype) tt_cores = (a.reshape(1, 10, rank_1), b, c.reshape((3, 8, 1))) # Basically do full by hand. desired = a.dot(b.reshape((rank_1, -1))) @@ -76,7 +76,8 @@ def testFlatInnerTTTensbySparseTens(self): sparse_flat_indices = sparse_flat_indices.astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, shape) sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(num_elements).astype(self.np_dtype) + values = np.random.randn(num_elements) + values = values.astype(self.tf_dtype.as_numpy_dtype) sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, dense_shape=shape) res_actual = ops.flat_inner(tt_1, sparse_2) @@ -194,8 +195,8 @@ class _TTMatrixTest(): def testFullMatrix2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(2, 3, rank).astype(self.np_dtype) - b = np.random.rand(rank, 4, 5).astype(self.np_dtype) + a = np.random.rand(2, 3, rank).astype(self.tf_dtype.as_numpy_dtype) + b = np.random.rand(rank, 4, 5).astype(self.tf_dtype.as_numpy_dtype) tt_cores = (a.reshape(1, 2, 3, rank), b.reshape((rank, 4, 5, 1))) # Basically do full by hand. desired = a.reshape((-1, rank)).dot(b.reshape((rank, -1))) @@ -210,9 +211,9 @@ def testFullMatrix2d(self): def testFullMatrix3d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(2, 3, rank).astype(self.np_dtype) - b = np.random.rand(rank, 4, 5, rank).astype(self.np_dtype) - c = np.random.rand(rank, 2, 2).astype(self.np_dtype) + a = np.random.rand(2, 3, rank).astype(self.tf_dtype.as_numpy_dtype) + b = np.random.rand(rank, 4, 5, rank).astype(self.tf_dtype.as_numpy_dtype) + c = np.random.rand(rank, 2, 2).astype(self.tf_dtype.as_numpy_dtype) tt_cores = (a.reshape(1, 2, 3, rank), b.reshape(rank, 4, 5, rank), c.reshape(rank, 2, 2, 1)) # Basically do full by hand. @@ -248,7 +249,7 @@ def testTTMatTimesDenseVec(self): inp_shape = (2, 3, 4) out_shape = (3, 4, 3) np.random.seed(1) - vec = np.random.rand(np.prod(inp_shape), 1).astype(self.np_dtype) + vec = np.random.rand(np.prod(inp_shape), 1).astype(self.tf_dtype.as_numpy_dtype) with self.test_session() as sess: tf_vec = tf.constant(vec) tf.set_random_seed(1) @@ -265,7 +266,7 @@ def testDenseMatTimesTTVec(self): out_shape = (3, 3, 3, 3) np.random.seed(1) mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) - mat = mat.astype(self.np_dtype) + mat = mat.astype(self.tf_dtype.as_numpy_dtype) with self.test_session() as sess: tf_mat = tf.constant(mat) tf.set_random_seed(1) @@ -313,7 +314,7 @@ def testFlatInnerTTMatbySparseMat(self): sparse_flat_indices = sparse_flat_indices.astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, matrix_shape) sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(num_elements).astype(self.np_dtype) + values = np.random.randn(num_elements).astype(self.tf_dtype.as_numpy_dtype) sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, dense_shape=matrix_shape) res_actual = ops.flat_inner(tt_1, sparse_2) @@ -467,8 +468,8 @@ class _TTTensorBatchTest(): def testFullTensor2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(3, 10, rank).astype(self.np_dtype) - b = np.random.rand(3, rank, 9).astype(self.np_dtype) + a = np.random.rand(3, 10, rank).astype(self.tf_dtype.as_numpy_dtype) + b = np.random.rand(3, rank, 9).astype(self.tf_dtype.as_numpy_dtype) tt_cores = (a.reshape(3, 1, 10, rank), b.reshape(3, rank, 9, 1)) desired = np.einsum('oib,obj->oij', a, b) with self.test_session(): @@ -479,9 +480,9 @@ def testFullTensor2d(self): def testFullTensor3d(self): np.random.seed(1) for rank_1 in [1, 2]: - a = np.random.rand(3, 10, rank_1).astype(self.np_dtype) - b = np.random.rand(3, rank_1, 9, 3).astype(self.np_dtype) - c = np.random.rand(3, 3, 8).astype(self.np_dtype) + a = np.random.rand(3, 10, rank_1).astype(self.tf_dtype.as_numpy_dtype) + b = np.random.rand(3, rank_1, 9, 3).astype(self.tf_dtype.as_numpy_dtype) + c = np.random.rand(3, 3, 8).astype(self.tf_dtype.as_numpy_dtype) tt_cores = (a.reshape(3, 1, 10, rank_1), b, c.reshape((3, 3, 8, 1))) # Basically do full by hand. desired = np.einsum('oia,oajb,obk->oijk', a, b, c) @@ -768,8 +769,8 @@ class _TTMatrixTestBatch(): def testFullMatrix2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(3, 2, 3, rank).astype(self.np_dtype) - b = np.random.rand(3, rank, 4, 5).astype(self.np_dtype) + a = np.random.rand(3, 2, 3, rank).astype(self.tf_dtype.as_numpy_dtype) + b = np.random.rand(3, rank, 4, 5).astype(self.tf_dtype.as_numpy_dtype) tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape((3, rank, 4, 5, 1))) # Basically do full by hand. desired = np.einsum('oijb,obkl->oijkl', a, b) @@ -784,9 +785,9 @@ def testFullMatrix2d(self): def testFullMatrix3d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(3, 2, 3, rank).astype(self.np_dtype) - b = np.random.rand(3, rank, 4, 5, rank).astype(self.np_dtype) - c = np.random.rand(3, rank, 2, 2).astype(self.np_dtype) + a = np.random.rand(3, 2, 3, rank).astype(self.tf_dtype.as_numpy_dtype) + b = np.random.rand(3, rank, 4, 5, rank).astype(self.tf_dtype.as_numpy_dtype) + c = np.random.rand(3, rank, 2, 2).astype(self.tf_dtype.as_numpy_dtype) tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape(3, rank, 4, 5, rank), c.reshape(3, rank, 2, 2, 1)) # Basically do full by hand. @@ -917,49 +918,41 @@ def _random_sparse(shape, non_zeros): sparse_flat_indices = np.random.choice(np.prod(shape), non_zeros).astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, shape) sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(non_zeros).astype(self.np_dtype) + values = np.random.randn(non_zeros).astype(self.tf_dtype.as_numpy_dtype) sparse = tf.SparseTensor(indices=sparse_indices, values=values, dense_shape=shape) return sparse class TTTensorTestFloat32(tf.test.TestCase, _TTTensorTest): - np_dtype = np.float32 tf_dtype = tf.float32 class TTTensorTestFloat64(tf.test.TestCase, _TTTensorTest): - np_dtype = np.float64 tf_dtype = tf.float64 class TTMatrixTestFloat32(tf.test.TestCase, _TTMatrixTest): - np_dtype = np.float32 tf_dtype = tf.float32 class TTMatrixTestFloat64(tf.test.TestCase, _TTMatrixTest): - np_dtype = np.float64 tf_dtype = tf.float64 class TTTensorBatchTestFloat32(tf.test.TestCase, _TTTensorBatchTest): - np_dtype = np.float32 tf_dtype = tf.float32 class TTTensorBatchTestFloat64(tf.test.TestCase, _TTTensorBatchTest): - np_dtype = np.float64 tf_dtype = tf.float64 class TTMatrixTestBatchFloat32(tf.test.TestCase, _TTMatrixTestBatch): - np_dtype = np.float32 tf_dtype = tf.float32 class TTMatrixTestBatchFloat64(tf.test.TestCase, _TTMatrixTestBatch): - np_dtype = np.float64 tf_dtype = tf.float64 diff --git a/t3f/riemannian_test.py b/t3f/riemannian_test.py index dd1e3901..16752592 100644 --- a/t3f/riemannian_test.py +++ b/t3f/riemannian_test.py @@ -32,7 +32,7 @@ def testProject(self): [[-0.19279465], [ 0.524976 ], [-0.40149197]]]) - convert = lambda t: np.array(t, dtype=self.np_dtype) + convert = lambda t: np.array(t, dtype=self.tf_dtype.as_numpy_dtype) tangent_tens_cores = list([convert(t) for t in tangent_tens_cores]) tangent_tens = TensorTrain(tangent_tens_cores, (4, 3), (1, 2, 1)) tens_cores = ([[[-1.01761142, 0.36075896, -0.2493624 ], @@ -60,7 +60,7 @@ def testProject(self): with self.test_session() as sess: proj_v = proj_full.eval() self.assertAllClose(desired_projection, proj_v) - self.assertEqual(self.np_dtype, proj_v.dtype) + self.assertEqual(self.tf_dtype.as_numpy_dtype, proj_v.dtype) def testProjectSum(self): # Test projecting a batch of TT-tensors. @@ -98,7 +98,7 @@ def testProjectWeightedSumMultipleOutputs(self): tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4, dtype=self.tf_dtype) np.random.seed(0) - weights = np.random.randn(4, 2).astype(self.np_dtype) + weights = np.random.randn(4, 2).astype(self.tf_dtype.as_numpy_dtype) tangent_tens = initializers.random_tensor((2, 3, 4), 4, dtype=self.tf_dtype) weighted_sum_1 = weights[0, 0] * tens[0] + weights[1, 0] * tens[1] +\ @@ -276,12 +276,10 @@ def testWeightedAddNProjectedBatch(self): class RiemannianTestFloat32(tf.test.TestCase, _RiemannianTest): - np_dtype = np.float32 tf_dtype = tf.float32 class RiemannianTestFloat64(tf.test.TestCase, _RiemannianTest): - np_dtype = np.float64 tf_dtype = tf.float64 diff --git a/t3f/tensor_train_batch_test.py b/t3f/tensor_train_batch_test.py index 98f8cdb3..78bb8bd0 100644 --- a/t3f/tensor_train_batch_test.py +++ b/t3f/tensor_train_batch_test.py @@ -84,12 +84,10 @@ def testShapeOverflow(self): class TensorTrainBatchTestFloat32(tf.test.TestCase, _TensorTrainBatchTest): - np_dtype = np.float32 tf_dtype = tf.float32 class TensorTrainBatchTestFloat64(tf.test.TestCase, _TensorTrainBatchTest): - np_dtype = np.float64 tf_dtype = tf.float64 diff --git a/t3f/tensor_train_test.py b/t3f/tensor_train_test.py index d22f9c93..e7b37fc0 100644 --- a/t3f/tensor_train_test.py +++ b/t3f/tensor_train_test.py @@ -144,12 +144,10 @@ def testShapeOverflow(self): class TensorTrainTestFloat32(tf.test.TestCase, _TensorTrainTest): - np_dtype = np.float32 tf_dtype = tf.float32 class TensorTrainTestFloat64(tf.test.TestCase, _TensorTrainTest): - np_dtype = np.float64 tf_dtype = tf.float64 diff --git a/t3f/variables_test.py b/t3f/variables_test.py index 3d7c2d6c..aec65b71 100644 --- a/t3f/variables_test.py +++ b/t3f/variables_test.py @@ -84,12 +84,10 @@ def testAssign(self): class VariablesTestFloat32(tf.test.TestCase, _VariablesTest): - np_dtype = np.float32 tf_dtype = tf.float32 class VariablesTestFloat64(tf.test.TestCase, _VariablesTest): - np_dtype = np.float64 tf_dtype = tf.float64 From 533ff9cab2edeb6d02987465ca316bad1412754c Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 20:29:36 +0000 Subject: [PATCH 092/233] s/tf_dtype/dtype/ --- t3f/approximate_test.py | 16 +-- t3f/batch_ops_test.py | 36 ++--- t3f/decompositions_test.py | 36 ++--- t3f/initializers_test.py | 48 +++---- t3f/kronecker_test.py | 38 +++--- t3f/ops_test.py | 240 ++++++++++++++++----------------- t3f/riemannian_test.py | 74 +++++----- t3f/tensor_train_batch_test.py | 10 +- t3f/tensor_train_test.py | 20 +-- t3f/variables_test.py | 20 +-- 10 files changed, 269 insertions(+), 269 deletions(-) diff --git a/t3f/approximate_test.py b/t3f/approximate_test.py index 41d8e06c..9082bf20 100644 --- a/t3f/approximate_test.py +++ b/t3f/approximate_test.py @@ -11,9 +11,9 @@ class _ApproximateTest(): def testAddN(self): # Sum a bunch of TT-matrices. tt_a = initializers.random_matrix(((2, 1, 4), (2, 2, 2)), tt_rank=2, - dtype=self.tf_dtype) + dtype=self.dtype) tt_b = initializers.random_matrix(((2, 1, 4), (2, 2, 2)), - tt_rank=[1, 2, 4, 1], dtype=self.tf_dtype) + tt_rank=[1, 2, 4, 1], dtype=self.dtype) def desired(tt_objects): res = tt_objects[0] @@ -50,7 +50,7 @@ def desired(tt_batch): tt_batch = initializers.random_tensor_batch((4, 3, 5), tt_rank=2, batch_size=batch_size, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 10)) res_desired = ops.full(desired(tt_batch)) res_desired_val, res_actual_val = sess.run([res_desired, res_actual]) @@ -68,7 +68,7 @@ def desired(tt_batch, coef): tt_batch = initializers.random_tensor_batch((4, 3, 5), tt_rank=3, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 9, [1.2, -0.2, 1])) res_desired = ops.full(desired(tt_batch, [1.2, -0.2, 1])) @@ -86,11 +86,11 @@ def desired(tt_batch, coef): with self.test_session() as sess: tt_batch = initializers.random_tensor_batch((4, 3, 5), tt_rank=2, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) coef = [[1., 0.1], [0.9, -0.2], [0.3, 0.3]] - coef = np.array(coef).astype(self.tf_dtype.as_numpy_dtype) + coef = np.array(coef).astype(self.dtype.as_numpy_dtype) res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 6, coef)) res_desired_1 = ops.full(desired(tt_batch, coef[:, 0])) @@ -101,11 +101,11 @@ def desired(tt_batch, coef): class ApproximateTestFloat32(tf.test.TestCase, _ApproximateTest): - tf_dtype = tf.float32 + dtype = tf.float32 class ApproximateTestFloat64(tf.test.TestCase, _ApproximateTest): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": diff --git a/t3f/batch_ops_test.py b/t3f/batch_ops_test.py index 8abc9dcc..1cc1de53 100644 --- a/t3f/batch_ops_test.py +++ b/t3f/batch_ops_test.py @@ -11,11 +11,11 @@ class _BatchOpsTest(): def testConcatMatrix(self): # Test concating TTMatrix batches along batch dimension. first = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=1, - dtype=self.tf_dtype) + dtype=self.dtype) second = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=4, - dtype=self.tf_dtype) + dtype=self.dtype) third = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) first_res = batch_ops.concat_along_batch_dim((first)) first_res = ops.full(first_res) first_second_res = batch_ops.concat_along_batch_dim((first, second)) @@ -49,7 +49,7 @@ def testConcatTensorPlaceholders(self): # Test concating TTTensors of unknown batch sizes along batch dimension. number_of_objects = tf.placeholder(tf.int32) all = initializers.random_tensor_batch((2, 3), batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) actual = batch_ops.concat_along_batch_dim((all[:number_of_objects], all[number_of_objects:])) with self.test_session() as sess: @@ -61,7 +61,7 @@ def testConcatMatrixPlaceholders(self): # Test concating TTMatrices of unknown batch sizes along batch dimension. number_of_objects = tf.placeholder(tf.int32) all = initializers.random_matrix_batch(((2, 3), (2, 3)), batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) actual = batch_ops.concat_along_batch_dim((all[:number_of_objects], all[number_of_objects:])) with self.test_session() as sess: @@ -72,7 +72,7 @@ def testConcatMatrixPlaceholders(self): def testBatchMultiply(self): # Test multiplying batch of TTMatrices by individual numbers. tt = initializers.random_matrix_batch(((2, 3), (3, 3)), batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) weights = [0.1, 0, -10] actual = batch_ops.multiply_along_batch_dim(tt, weights) individual_desired = [weights[i] * tt[i:i+1] for i in range(3)] @@ -84,7 +84,7 @@ def testBatchMultiply(self): def testGramMatrix(self): # Test Gram Matrix of a batch of TT vectors. tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = batch_ops.gram_matrix(tt_vectors) full_vectors = tf.reshape(ops.full(tt_vectors), (5, 6)) res_desired = tf.matmul(full_vectors, tf.transpose(full_vectors)) @@ -98,8 +98,8 @@ def testGramMatrixWithMatrix(self): # should compute # res[i, j] = tt_vectors[i] ^ T * matrix * tt_vectors[j] tt_vectors = initializers.random_matrix_batch(((2, 3), None), batch_size=4, - dtype=self.tf_dtype) - matrix = initializers.random_matrix(((2, 3), (2, 3)), dtype=self.tf_dtype) + dtype=self.dtype) + matrix = initializers.random_matrix(((2, 3), (2, 3)), dtype=self.dtype) res_actual = batch_ops.gram_matrix(tt_vectors, matrix) full_vectors = tf.reshape(ops.full(tt_vectors), (4, 6)) with self.test_session() as sess: @@ -116,9 +116,9 @@ def testGramMatrixWithMatrix(self): def testPairwiseFlatInnerTensor(self): # Test pairwise_flat_inner of a batch of TT tensors. tt_tensors_1 = initializers.random_tensor_batch((2, 3, 2), batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) tt_tensors_2 = initializers.random_tensor_batch((2, 3, 2), batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = batch_ops.pairwise_flat_inner(tt_tensors_1, tt_tensors_2) full_tensors_1 = tf.reshape(ops.full(tt_tensors_1), (5, 12)) full_tensors_2 = tf.reshape(ops.full(tt_tensors_2), (5, 12)) @@ -132,10 +132,10 @@ def testPairwiseFlatInnerMatrix(self): # Test pairwise_flat_inner of a batch of TT matrices. tt_vectors_1 = initializers.random_matrix_batch(((2, 3), (2, 3)), batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) tt_vectors_2 = initializers.random_matrix_batch(((2, 3), (2, 3)), batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = batch_ops.pairwise_flat_inner(tt_vectors_1, tt_vectors_2) full_vectors_1 = tf.reshape(ops.full(tt_vectors_1), (5, 36)) full_vectors_2 = tf.reshape(ops.full(tt_vectors_2), (5, 36)) @@ -151,11 +151,11 @@ def testPairwiseFlatInnerVectorsWithMatrix(self): # res[i, j] = tt_vectors[i] ^ T * matrix * tt_vectors[j] tt_vectors_1 = initializers.random_matrix_batch(((2, 3), None), batch_size=2, - dtype=self.tf_dtype) + dtype=self.dtype) tt_vectors_2 = initializers.random_matrix_batch(((2, 3), None), batch_size=3, - dtype=self.tf_dtype) - matrix = initializers.random_matrix(((2, 3), (2, 3)), dtype=self.tf_dtype) + dtype=self.dtype) + matrix = initializers.random_matrix(((2, 3), (2, 3)), dtype=self.dtype) res_actual = batch_ops.pairwise_flat_inner(tt_vectors_1, tt_vectors_2, matrix) full_vectors_1 = tf.reshape(ops.full(tt_vectors_1), (2, 6)) @@ -174,11 +174,11 @@ def testPairwiseFlatInnerVectorsWithMatrix(self): class BatchOpsTestFloat32(tf.test.TestCase, _BatchOpsTest): - tf_dtype = tf.float32 + dtype = tf.float32 class BatchOpsTestFloat64(tf.test.TestCase, _BatchOpsTest): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": diff --git a/t3f/decompositions_test.py b/t3f/decompositions_test.py index 7b2ce807..4d87dead 100644 --- a/t3f/decompositions_test.py +++ b/t3f/decompositions_test.py @@ -12,7 +12,7 @@ class _DecompositionsTest(): def testTTTensor(self): shape = (2, 1, 4, 3) np.random.seed(1) - tens = np.random.rand(*shape).astype(self.tf_dtype.as_numpy_dtype) + tens = np.random.rand(*shape).astype(self.dtype.as_numpy_dtype) tf_tens = tf.constant(tens) tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=3) with self.test_session(): @@ -22,7 +22,7 @@ def testTTTensor(self): self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) # Try to decompose the same tensor with unknown shape. - tf_tens_pl = tf.placeholder(self.tf_dtype, (None, None, 4, None)) + tf_tens_pl = tf.placeholder(self.dtype, (None, None, 4, None)) tt_tens = decompositions.to_tt_tensor(tf_tens_pl, max_tt_rank=3) tt_val = ops.full(tt_tens).eval({tf_tens_pl: tens}) self.assertAllClose(tens, tt_val) @@ -33,8 +33,8 @@ def testTTTensorSimple(self): # Test that a tensor of ones and of zeros can be converted into TT with # TT-rank 1. shape = (2, 1, 4, 3) - tens_arr = (np.zeros(shape).astype(self.tf_dtype.as_numpy_dtype), - np.ones(shape).astype(self.tf_dtype.as_numpy_dtype)) + tens_arr = (np.zeros(shape).astype(self.dtype.as_numpy_dtype), + np.ones(shape).astype(self.dtype.as_numpy_dtype)) for tens in tens_arr: tf_tens = tf.constant(tens) tt_tens = decompositions.to_tt_tensor(tf_tens, max_tt_rank=1) @@ -45,7 +45,7 @@ def testTTTensorSimple(self): self.assertAllEqual(dynamic_tt_ranks, static_tt_ranks) # Try to decompose the same tensor with unknown shape. - tf_tens_pl = tf.placeholder(self.tf_dtype, (None, None, None, None)) + tf_tens_pl = tf.placeholder(self.dtype, (None, None, None, None)) tt_tens = decompositions.to_tt_tensor(tf_tens_pl, max_tt_rank=1) tt_val = ops.full(tt_tens).eval({tf_tens_pl: tens}) self.assertAllClose(tens, tt_val) @@ -56,7 +56,7 @@ def testTTVector(self): vec_shape = (2, 1, 4, 3) np.random.seed(1) rows = np.prod(vec_shape) - vec = np.random.rand(rows, 1).astype(self.tf_dtype.as_numpy_dtype) + vec = np.random.rand(rows, 1).astype(self.dtype.as_numpy_dtype) tf_vec = tf.constant(vec) tt_vec = decompositions.to_tt_matrix(tf_vec, (vec_shape, None)) with self.test_session(): @@ -66,7 +66,7 @@ def testTTCompositeRankTensor(self): # Test if a composite rank (list of ranks) can be used for decomposition # for tensor. np.random.seed(1) - np_tensor = np.random.rand(2, 3, 3, 1).astype(self.tf_dtype.as_numpy_dtype) + np_tensor = np.random.rand(2, 3, 3, 1).astype(self.dtype.as_numpy_dtype) tf_tensor = tf.constant(np_tensor) tt_ranks = [1, 2, 3, 3, 1] @@ -81,7 +81,7 @@ def testTTCompositeRankMatrix(self): out_shape = (1, 2, 2, 1) np.random.seed(1) mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) - mat = mat.astype(self.tf_dtype.as_numpy_dtype) + mat = mat.astype(self.dtype.as_numpy_dtype) tf_mat = tf.constant(mat) tt_ranks = [10, 20, 30, 40, 30] tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), @@ -96,7 +96,7 @@ def testTTMatrix(self): out_shape = (3, 3, 2, 3) np.random.seed(1) mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) - mat = mat.astype(self.tf_dtype.as_numpy_dtype) + mat = mat.astype(self.dtype.as_numpy_dtype) tf_mat = tf.constant(mat) tt_mat = decompositions.to_tt_matrix(tf_mat, (out_shape, inp_shape), max_tt_rank=90) @@ -108,7 +108,7 @@ def testRoundTensor(self): shape = (2, 1, 4, 3, 3) np.random.seed(1) tens = initializers.random_tensor(shape, tt_rank=15, - dtype=self.tf_dtype) + dtype=self.dtype) rounded_tens = decompositions.round(tens, max_tt_rank=9) with self.test_session() as sess: vars = [ops.full(tens), ops.full(rounded_tens)] @@ -123,7 +123,7 @@ def testOrthogonalizeLeftToRight(self): tt_ranks = (1, 5, 2, 17, 1) updated_tt_ranks = (1, 2, 2, 6, 1) tens = initializers.random_tensor(shape, tt_rank=tt_ranks, - dtype=self.tf_dtype) + dtype=self.dtype) orthogonal = decompositions.orthogonalize_tt_cores(tens) with self.test_session() as sess: tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) @@ -145,7 +145,7 @@ def testOrthogonalizeRightToLeft(self): tt_ranks = (1, 5, 2, 17, 1) updated_tt_ranks = (1, 5, 2, 3, 1) tens = initializers.random_tensor(shape, tt_rank=tt_ranks, - dtype=self.tf_dtype) + dtype=self.dtype) orthogonal = decompositions.orthogonalize_tt_cores(tens, left_to_right=False) with self.test_session() as sess: tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) @@ -170,7 +170,7 @@ def testOrthogonalizeLeftToRight(self): tt_ranks = (1, 5, 2, 17, 1) updated_tt_ranks = (1, 2, 2, 6, 1) tens = initializers.random_tensor_batch(shape, tt_rank=tt_ranks, - batch_size=2, dtype=self.tf_dtype) + batch_size=2, dtype=self.dtype) orthogonal = decompositions.orthogonalize_tt_cores(tens) with self.test_session() as sess: tens_val, orthogonal_val = sess.run([ops.full(tens), ops.full(orthogonal)]) @@ -191,7 +191,7 @@ def testOrthogonalizeLeftToRight(self): def testRoundTensor(self): shape = (2, 1, 4, 3, 3) tens = initializers.random_tensor_batch(shape, tt_rank=15, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) rounded_tens = decompositions.round(tens, max_tt_rank=9) with self.test_session() as sess: vars = [ops.full(tens), ops.full(rounded_tens)] @@ -204,19 +204,19 @@ def testRoundTensor(self): class DecompositionsTestFloat32(tf.test.TestCase, _DecompositionsTest): - tf_dtype = tf.float32 + dtype = tf.float32 class DecompositionsTestFloat64(tf.test.TestCase, _DecompositionsTest): - tf_dtype = tf.float64 + dtype = tf.float64 class DecompositionsBatchTestFloat32(tf.test.TestCase, _DecompositionsBatchTest): - tf_dtype = tf.float32 + dtype = tf.float32 class DecompositionsBatchTestFloat64(tf.test.TestCase, _DecompositionsBatchTest): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": diff --git a/t3f/initializers_test.py b/t3f/initializers_test.py index 8b30ca94..eae3173c 100644 --- a/t3f/initializers_test.py +++ b/t3f/initializers_test.py @@ -8,11 +8,11 @@ class _InitializersTest(): def testTensorOnesAndZeros(self): - tt_ones = initializers.tensor_ones([2, 3, 4], dtype=self.tf_dtype) - tt_zeros = initializers.tensor_zeros([2, 3, 4], dtype=self.tf_dtype) + tt_ones = initializers.tensor_ones([2, 3, 4], dtype=self.dtype) + tt_zeros = initializers.tensor_zeros([2, 3, 4], dtype=self.dtype) - ones_desired = np.ones((2, 3, 4), dtype=self.tf_dtype.as_numpy_dtype) - zeros_desired = np.zeros((2, 3, 4), dtype=self.tf_dtype.as_numpy_dtype) + ones_desired = np.ones((2, 3, 4), dtype=self.dtype.as_numpy_dtype) + zeros_desired = np.zeros((2, 3, 4), dtype=self.dtype.as_numpy_dtype) with self.test_session() as sess: tt_ones_full = sess.run(ops.full(tt_ones)) tt_zeros_full = sess.run(ops.full(tt_zeros)) @@ -29,12 +29,12 @@ def testTensorOnesAndZeros(self): def testMatrixOnesAndZeros(self): tt_ones = initializers.matrix_ones([[2, 3, 4], [1, 2, 5]], - dtype=self.tf_dtype) + dtype=self.dtype) tt_zeros = initializers.matrix_zeros([[2, 3, 4], [1, 2, 5]], - dtype=self.tf_dtype) + dtype=self.dtype) - ones_desired = np.ones((24, 10), dtype=self.tf_dtype.as_numpy_dtype) - zeros_desired = np.zeros((24, 10), dtype=self.tf_dtype.as_numpy_dtype) + ones_desired = np.ones((24, 10), dtype=self.dtype.as_numpy_dtype) + zeros_desired = np.zeros((24, 10), dtype=self.dtype.as_numpy_dtype) bad_shapes = [[[-1, 2, 3], [3, 4, 6]], [[1.5, 2, 4], [2, 5, 6]], [[1], [2, 3]], [2, 3, 4]] @@ -52,7 +52,7 @@ def testMatrixOnesAndZeros(self): initializers.matrix_zeros(shape) def testEye(self): - tt_eye = initializers.eye([4, 5, 6], dtype=self.tf_dtype) + tt_eye = initializers.eye([4, 5, 6], dtype=self.dtype) eye_desired = np.eye(120) with self.test_session() as sess: eye_full = sess.run(ops.full(tt_eye)) @@ -63,16 +63,16 @@ def testEye(self): initializers.eye(shape) def testOnesLikeAndZerosLike(self): - a = initializers.random_tensor([2, 3, 4], dtype=self.tf_dtype) + a = initializers.random_tensor([2, 3, 4], dtype=self.dtype) b = initializers.ones_like(a) c = initializers.zeros_like(a) var_list = [ops.full(b), ops.full(c)] with self.test_session() as sess: bf, cf = sess.run(var_list) self.assertAllClose(bf, np.ones((2, 3, 4))) - self.assertEqual(self.tf_dtype.as_numpy_dtype, bf.dtype) + self.assertEqual(self.dtype.as_numpy_dtype, bf.dtype) self.assertAllClose(cf, np.zeros((2, 3, 4))) - self.assertEqual(self.tf_dtype.as_numpy_dtype, cf.dtype) + self.assertEqual(self.dtype.as_numpy_dtype, cf.dtype) with self.assertRaises(ValueError): initializers.ones_like(1) with self.assertRaises(ValueError): @@ -85,12 +85,12 @@ def testInvalidShapeOrRanksTensor(self): for case in bad_cases: with self.assertRaises(ValueError): initializers.random_tensor(case[0], tt_rank=case[1], - dtype=self.tf_dtype) + dtype=self.dtype) for case in bad_cases: with self.assertRaises(ValueError): initializers.tensor_with_random_cores(case[0], tt_rank=case[1], - dtype=self.tf_dtype) + dtype=self.dtype) def testInvalidShapeOrRanksMatrix(self): shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], @@ -102,11 +102,11 @@ def testInvalidShapeOrRanksMatrix(self): for case in bad_cases: with self.assertRaises(ValueError): initializers.random_matrix(case[0], tt_rank=case[1], - dtype=self.tf_dtype) + dtype=self.dtype) for case in bad_cases: with self.assertRaises(ValueError): initializers.matrix_with_random_cores(case[0], tt_rank=case[1], - dtype=self.tf_dtype) + dtype=self.dtype) def testInvalidShapeOrRanksTensorBatch(self): shapes = [[3, 4], [3, 4], [3, 4], [3, 4], [1, -2], [1.1, 2], [[3, 4]], @@ -118,12 +118,12 @@ def testInvalidShapeOrRanksTensorBatch(self): with self.assertRaises(ValueError): initializers.random_tensor_batch(case[0], tt_rank=case[1], batch_size=case[2], - dtype=self.tf_dtype) + dtype=self.dtype) for case in bad_cases: with self.assertRaises(ValueError): initializers.tensor_batch_with_random_cores(case[0], tt_rank=case[1], batch_size=case[2], - dtype=self.tf_dtype) + dtype=self.dtype) def testInvalidShapeOrRanksMatrixBatch(self): shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], @@ -138,12 +138,12 @@ def testInvalidShapeOrRanksMatrixBatch(self): with self.assertRaises(ValueError): initializers.random_matrix_batch(case[0], tt_rank=case[1], batch_size=case[2], - dtype=self.tf_dtype) + dtype=self.dtype) for case in bad_cases: with self.assertRaises(ValueError): initializers.matrix_batch_with_random_cores(case[0], tt_rank=case[1], batch_size=case[2], - dtype=self.tf_dtype) + dtype=self.dtype) def testInvalidShapeOrRanksGlorot(self): shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], @@ -166,7 +166,7 @@ def testHe(self): for case in bad_cases: with self.assertRaises(ValueError): initializers.he_initializer(case[0], tt_rank=case[1], - dtype=self.tf_dtype) + dtype=self.dtype) def testInvalidShapeOrRanksLecun(self): shapes = [[1, 2, 3], [[1, 2], [1, 2, 3]], [[-1, 2, 3], [1, 2, 3]], @@ -178,15 +178,15 @@ def testInvalidShapeOrRanksLecun(self): for case in bad_cases: with self.assertRaises(ValueError): initializers.lecun_initializer(case[0], tt_rank=case[1], - dtype=self.tf_dtype) + dtype=self.dtype) class InitializersTestFloat32(tf.test.TestCase, _InitializersTest): - tf_dtype = tf.float32 + dtype = tf.float32 class InitializersTestFloat64(tf.test.TestCase, _InitializersTest): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": diff --git a/t3f/kronecker_test.py b/t3f/kronecker_test.py index ebc1d257..85112cf9 100644 --- a/t3f/kronecker_test.py +++ b/t3f/kronecker_test.py @@ -13,21 +13,21 @@ class _KroneckerTest(): def testIsKronNonKron(self): # Tests _is_kron on a non-Kronecker matrix initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=2, - dtype=self.tf_dtype) + dtype=self.dtype) tt_mat = variables.get_variable('tt_mat', initializer=initializer) self.assertFalse(kr._is_kron(tt_mat)) def testIsKronKron(self): # Tests _is_kron on a Kronecker matrix initializer = initializers.random_matrix(((2, 3), (3, 2)), tt_rank=1, - dtype=self.tf_dtype) + dtype=self.dtype) kron_mat = variables.get_variable('kron_mat', initializer=initializer) self.assertTrue(kr._is_kron(kron_mat)) def testDet(self): # Tests the determinant function initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1, - dtype=self.tf_dtype) + dtype=self.dtype) kron_mat = variables.get_variable('kron_mat', initializer=initializer) init_op = tf.global_variables_initializer() with self.test_session() as sess: @@ -44,12 +44,12 @@ def testSlogDet(self): tf.set_random_seed(5) # negative derminant initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1, - dtype=self.tf_dtype) + dtype=self.dtype) kron_neg = variables.get_variable('kron_neg', initializer=initializer) tf.set_random_seed(1) # positive determinant initializer = initializers.random_matrix(((2, 3), (2, 3)), tt_rank=1, - dtype=self.tf_dtype) + dtype=self.dtype) kron_pos = variables.get_variable('kron_pos', initializer=initializer) init_op = tf.global_variables_initializer() @@ -70,7 +70,7 @@ def testSlogDet(self): def testInv(self): # Tests the inv function initializer = initializers.random_matrix(((2, 3, 2), (2, 3, 2)), tt_rank=1, - dtype=self.tf_dtype) + dtype=self.dtype) kron_mat = variables.get_variable('kron_mat', initializer=initializer) init_op = tf.global_variables_initializer() with self.test_session() as sess: @@ -85,9 +85,9 @@ def testCholesky(self): # generating two symmetric positive-definite tt-cores L_1 = np.tril(np.random.normal(scale=2., size=(2, 2))) - L_1 = L_1.astype(self.tf_dtype.as_numpy_dtype) + L_1 = L_1.astype(self.dtype.as_numpy_dtype) L_2 = np.tril(np.random.normal(scale=2., size=(3, 3))) - L_2 = L_2.astype(self.tf_dtype.as_numpy_dtype) + L_2 = L_2.astype(self.dtype.as_numpy_dtype) K_1 = L_1.dot(L_1.T) K_2 = L_2.dot(L_2.T) K = np.kron(K_1, K_2) @@ -108,7 +108,7 @@ def testIsKronNonKron(self): # Tests _is_kron on a non-Kronecker matrix batch initializer = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=2, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) tt_mat_batch = variables.get_variable('tt_mat_batch', initializer=initializer) self.assertFalse(kr._is_kron(tt_mat_batch)) @@ -117,7 +117,7 @@ def testIsKronKron(self): # Tests _is_kron on a Kronecker matrix batch initializer = initializers.random_matrix_batch(((2, 3), (3, 2)), tt_rank=1, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) kron_mat_batch = variables.get_variable('kron_mat_batch', initializer=initializer) self.assertTrue(kr._is_kron(kron_mat_batch)) @@ -126,7 +126,7 @@ def testDet(self): # Tests the determinant function initializer = initializers.random_matrix_batch(((2, 3, 2), (2, 3, 2)), tt_rank=1, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) kron_mat_batch = variables.get_variable('kron_mat_batch', initializer=initializer) init_op = tf.global_variables_initializer() @@ -142,7 +142,7 @@ def testSlogDet(self): tf.set_random_seed(1) # negative and positive determinants initializer = initializers.random_matrix_batch(((2, 3), (2, 3)), tt_rank=1, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) kron_mat_batch = variables.get_variable('kron_mat_batch', initializer=initializer) @@ -160,7 +160,7 @@ def testInv(self): # Tests the inv function initializer = initializers.random_matrix_batch(((2, 3, 2), (2, 3, 2)), tt_rank=1, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) kron_mat_batch = variables.get_variable('kron_mat_batch', initializer=initializer) init_op = tf.global_variables_initializer() @@ -176,9 +176,9 @@ def testCholesky(self): # generating two symmetric positive-definite tt-cores L_1 = np.tril(np.random.normal(scale=2., size=(4, 2, 2))) - L_1 = L_1.astype(self.tf_dtype.as_numpy_dtype) + L_1 = L_1.astype(self.dtype.as_numpy_dtype) L_2 = np.tril(np.random.normal(scale=2., size=(4, 3, 3))) - L_2 = L_2.astype(self.tf_dtype.as_numpy_dtype) + L_2 = L_2.astype(self.dtype.as_numpy_dtype) K_1 = np.einsum('ijk,ilk->ijl', L_1, L_1) K_2 = np.einsum('ijk,ilk->ijl', L_2, L_2) initializer = TensorTrainBatch([K_1[:, None, :, :, None], @@ -194,19 +194,19 @@ def testCholesky(self): class KroneckerTestFloat32(tf.test.TestCase, _KroneckerTest): - tf_dtype = tf.float32 + dtype = tf.float32 class KroneckerTestFloat64(tf.test.TestCase, _KroneckerTest): - tf_dtype = tf.float64 + dtype = tf.float64 class BatchKroneckerTestFloat32(tf.test.TestCase, _BatchKroneckerTest): - tf_dtype = tf.float32 + dtype = tf.float32 class BatchKroneckerTestFloat64(tf.test.TestCase, _BatchKroneckerTest): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": diff --git a/t3f/ops_test.py b/t3f/ops_test.py index 775fd3ce..51c21900 100644 --- a/t3f/ops_test.py +++ b/t3f/ops_test.py @@ -13,8 +13,8 @@ class _TTTensorTest(): def testFullTensor2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(10, rank).astype(self.tf_dtype.as_numpy_dtype) - b = np.random.rand(rank, 9).astype(self.tf_dtype.as_numpy_dtype) + a = np.random.rand(10, rank).astype(self.dtype.as_numpy_dtype) + b = np.random.rand(rank, 9).astype(self.dtype.as_numpy_dtype) tt_cores = (a.reshape(1, 10, rank), b.reshape(rank, 9, 1)) desired = np.dot(a, b) with self.test_session(): @@ -25,9 +25,9 @@ def testFullTensor2d(self): def testFullTensor3d(self): np.random.seed(1) for rank_1 in [1, 2]: - a = np.random.rand(10, rank_1).astype(self.tf_dtype.as_numpy_dtype) - b = np.random.rand(rank_1, 9, 3).astype(self.tf_dtype.as_numpy_dtype) - c = np.random.rand(3, 8).astype(self.tf_dtype.as_numpy_dtype) + a = np.random.rand(10, rank_1).astype(self.dtype.as_numpy_dtype) + b = np.random.rand(rank_1, 9, 3).astype(self.dtype.as_numpy_dtype) + c = np.random.rand(3, 8).astype(self.dtype.as_numpy_dtype) tt_cores = (a.reshape(1, 10, rank_1), b, c.reshape((3, 8, 1))) # Basically do full by hand. desired = a.dot(b.reshape((rank_1, -1))) @@ -48,9 +48,9 @@ def testFlatInnerTTTensbyTTTens(self): for shape in shape_list: for rank in rank_list: tt_1 = initializers.random_tensor(shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) tt_2 = initializers.random_tensor(shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.flat_inner(tt_1, tt_2) tt_1_full = tf.reshape(ops.full(tt_1), (1, -1)) tt_2_full = tf.reshape(ops.full(tt_2), (-1, 1)) @@ -71,13 +71,13 @@ def testFlatInnerTTTensbySparseTens(self): for rank in rank_list: for num_elements in [1, 10]: tt_1 = initializers.random_tensor(shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) sparse_flat_indices = np.random.choice(np.prod(shape), num_elements) sparse_flat_indices = sparse_flat_indices.astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, shape) sparse_indices = np.vstack(sparse_indices).transpose() values = np.random.randn(num_elements) - values = values.astype(self.tf_dtype.as_numpy_dtype) + values = values.astype(self.dtype.as_numpy_dtype) sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, dense_shape=shape) res_actual = ops.flat_inner(tt_1, sparse_2) @@ -88,9 +88,9 @@ def testFlatInnerTTTensbySparseTens(self): def testAdd(self): # Sum two TT-tensors. tt_a = initializers.random_tensor((2, 1, 3, 4), tt_rank=2, - dtype=self.tf_dtype) + dtype=self.dtype) tt_b = initializers.random_tensor((2, 1, 3, 4), tt_rank=[1, 2, 4, 3, 1], - dtype=self.tf_dtype) + dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_a + tt_b) @@ -103,9 +103,9 @@ def testAdd(self): def testMultiply(self): # Multiply two TT-tensors. tt_a = initializers.random_tensor((1, 2, 3, 4), tt_rank=2, - dtype=self.tf_dtype) + dtype=self.dtype) tt_b = initializers.random_tensor((1, 2, 3, 4), tt_rank=[1, 1, 4, 3, 1], - dtype=self.tf_dtype) + dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt_a, tt_b)) res_actual2 = ops.full(tt_a * tt_b) @@ -118,7 +118,7 @@ def testMultiply(self): def testMultiplyByNumber(self): # Multiply a tensor by a number. tt = initializers.random_tensor((1, 2, 3), tt_rank=(1, 2, 3, 1), - dtype=self.tf_dtype) + dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt, 4)) res_actual2 = ops.full(4.0 * tt) @@ -138,7 +138,7 @@ def testFrobeniusNormTens(self): for shape in shape_list: for rank in rank_list: tt = initializers.random_tensor(shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) norm_sq_actual = ops.frobenius_norm_squared(tt) norm_actual = ops.frobenius_norm(tt) vars = [norm_sq_actual, norm_actual, ops.full(tt)] @@ -155,10 +155,10 @@ def testCastFloat(self): tt_x = initializers.random_tensor((2, 3, 2), tt_rank=2) with self.test_session() as sess: - casted = ops.cast(tt_x, self.tf_dtype) + casted = ops.cast(tt_x, self.dtype) casted_val = sess.run(ops.full(casted)) - self.assertEqual(self.tf_dtype, casted.dtype) - self.assertTrue(self.tf_dtype, casted_val.dtype) + self.assertEqual(self.dtype, casted.dtype) + self.assertTrue(self.dtype, casted_val.dtype) def testCastIntFloat(self): # Tests cast function from int to float for tensors. @@ -169,14 +169,14 @@ def testCastIntFloat(self): tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) with self.test_session() as sess: - casted = ops.cast(tt_int, self.tf_dtype) + casted = ops.cast(tt_int, self.dtype) casted_val = sess.run(ops.full(casted)) - self.assertEqual(self.tf_dtype, casted.dtype) - self.assertTrue(self.tf_dtype, casted_val.dtype) + self.assertEqual(self.dtype, casted.dtype) + self.assertTrue(self.dtype, casted_val.dtype) def testCoreRenorm(self): a = initializers.random_tensor(3 * (10,), tt_rank=7, - dtype=self.tf_dtype) + dtype=self.dtype) b = ops.renormalize_tt_cores(a) var_list = [ops.full(a), ops.full(b)] with self.test_session() as sess: @@ -195,8 +195,8 @@ class _TTMatrixTest(): def testFullMatrix2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(2, 3, rank).astype(self.tf_dtype.as_numpy_dtype) - b = np.random.rand(rank, 4, 5).astype(self.tf_dtype.as_numpy_dtype) + a = np.random.rand(2, 3, rank).astype(self.dtype.as_numpy_dtype) + b = np.random.rand(rank, 4, 5).astype(self.dtype.as_numpy_dtype) tt_cores = (a.reshape(1, 2, 3, rank), b.reshape((rank, 4, 5, 1))) # Basically do full by hand. desired = a.reshape((-1, rank)).dot(b.reshape((rank, -1))) @@ -211,9 +211,9 @@ def testFullMatrix2d(self): def testFullMatrix3d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(2, 3, rank).astype(self.tf_dtype.as_numpy_dtype) - b = np.random.rand(rank, 4, 5, rank).astype(self.tf_dtype.as_numpy_dtype) - c = np.random.rand(rank, 2, 2).astype(self.tf_dtype.as_numpy_dtype) + a = np.random.rand(2, 3, rank).astype(self.dtype.as_numpy_dtype) + b = np.random.rand(rank, 4, 5, rank).astype(self.dtype.as_numpy_dtype) + c = np.random.rand(rank, 2, 2).astype(self.dtype.as_numpy_dtype) tt_cores = (a.reshape(1, 2, 3, rank), b.reshape(rank, 4, 5, rank), c.reshape(rank, 2, 2, 1)) # Basically do full by hand. @@ -234,9 +234,9 @@ def testTTMatTimesTTMat(self): right_shape = (4, 4, 4) with self.test_session() as sess: tt_mat_1 = initializers.random_matrix((left_shape, sum_shape), tt_rank=3, - dtype=self.tf_dtype) + dtype=self.dtype) tt_mat_2 = initializers.random_matrix((sum_shape, right_shape), - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.matmul(tt_mat_1, tt_mat_2) res_actual = ops.full(res_actual) res_desired = tf.matmul(ops.full(tt_mat_1), ops.full(tt_mat_2)) @@ -249,12 +249,12 @@ def testTTMatTimesDenseVec(self): inp_shape = (2, 3, 4) out_shape = (3, 4, 3) np.random.seed(1) - vec = np.random.rand(np.prod(inp_shape), 1).astype(self.tf_dtype.as_numpy_dtype) + vec = np.random.rand(np.prod(inp_shape), 1).astype(self.dtype.as_numpy_dtype) with self.test_session() as sess: tf_vec = tf.constant(vec) tf.set_random_seed(1) tt_mat = initializers.random_matrix((out_shape, inp_shape), - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.matmul(tt_mat, tf_vec) res_desired = tf.matmul(ops.full(tt_mat), tf_vec) res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) @@ -266,12 +266,12 @@ def testDenseMatTimesTTVec(self): out_shape = (3, 3, 3, 3) np.random.seed(1) mat = np.random.rand(np.prod(out_shape), np.prod(inp_shape)) - mat = mat.astype(self.tf_dtype.as_numpy_dtype) + mat = mat.astype(self.dtype.as_numpy_dtype) with self.test_session() as sess: tf_mat = tf.constant(mat) tf.set_random_seed(1) tt_vec = initializers.random_matrix((inp_shape, None), - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.matmul(tf_mat, tt_vec) res_desired = tf.matmul(tf_mat, ops.full(tt_vec)) res_actual_val, res_desired_val = sess.run([res_actual, res_desired]) @@ -286,9 +286,9 @@ def testFlatInnerTTMatbyTTMat(self): for shape in shape_list: for rank in rank_list: tt_1 = initializers.random_matrix(shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) tt_2 = initializers.random_matrix(shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.flat_inner(tt_1, tt_2) tt_1_full = tf.reshape(ops.full(tt_1), (1, -1)) tt_2_full = tf.reshape(ops.full(tt_2), (-1, 1)) @@ -308,13 +308,13 @@ def testFlatInnerTTMatbySparseMat(self): for rank in rank_list: for num_elements in [1, 9]: tt_1 = initializers.random_matrix(tensor_shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) matrix_shape = np.prod(tensor_shape[0]), np.prod(tensor_shape[1]) sparse_flat_indices = np.random.choice(np.prod(matrix_shape), num_elements) sparse_flat_indices = sparse_flat_indices.astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, matrix_shape) sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(num_elements).astype(self.tf_dtype.as_numpy_dtype) + values = np.random.randn(num_elements).astype(self.dtype.as_numpy_dtype) sparse_2 = tf.SparseTensor(indices=sparse_indices, values=values, dense_shape=matrix_shape) res_actual = ops.flat_inner(tt_1, sparse_2) @@ -331,7 +331,7 @@ def testFrobeniusNormMatrix(self): for tensor_shape in shape_list: for rank in rank_list: tt = initializers.random_matrix(tensor_shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) norm_sq_actual = ops.frobenius_norm_squared(tt) norm_actual = ops.frobenius_norm(tt) vars = [norm_sq_actual, norm_actual, ops.full(tt)] @@ -352,7 +352,7 @@ def testTranspose(self): for tensor_shape in shape_list: for rank in rank_list: tt = initializers.random_matrix(tensor_shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.full(ops.transpose(tt)) res_actual_val, tt_val = sess.run([res_actual, ops.full(tt)]) self.assertAllClose(tt_val.transpose(), res_actual_val) @@ -366,11 +366,11 @@ def testQuadraticForm(self): for tensor_shape in shape_list: for rank in rank_list: A = initializers.random_matrix(tensor_shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) b = initializers.random_matrix((tensor_shape[0], None), tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) c = initializers.random_matrix((tensor_shape[1], None), tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.quadratic_form(A, b, c) vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] res_actual_val, A_val, b_val, c_val = sess.run(vars) @@ -387,13 +387,13 @@ def testQuadraticFormBatch(self): for tensor_shape in shape_list: for rank in rank_list: A = initializers.random_matrix(tensor_shape, tt_rank=rank, - dtype=self.tf_dtype) + dtype=self.dtype) b = initializers.random_matrix_batch((tensor_shape[0], None), tt_rank=rank, batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) c = initializers.random_matrix_batch((tensor_shape[1], None), tt_rank=rank, batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.quadratic_form(A, b, c) vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] res_actual_val, A_val, b_val, c_val = sess.run(vars) @@ -409,10 +409,10 @@ def testCastFloat(self): with self.test_session() as sess: for tt in [tt_mat, tt_vec]: - casted = ops.cast(tt, self.tf_dtype) + casted = ops.cast(tt, self.dtype) casted_val = sess.run(ops.full(casted)) - self.assertEqual(self.tf_dtype, casted.dtype) - self.assertTrue(self.tf_dtype, casted_val.dtype) + self.assertEqual(self.dtype, casted.dtype) + self.assertTrue(self.dtype, casted_val.dtype) def testCastIntFloat(self): # Tests cast function from int to float for matrices. @@ -423,15 +423,15 @@ def testCastIntFloat(self): tt_int = TensorTrain([K_1, K_2, K_3], tt_ranks=[1, 2, 2, 1]) with self.test_session() as sess: - casted = ops.cast(tt_int, self.tf_dtype) + casted = ops.cast(tt_int, self.dtype) casted_val = sess.run(ops.full(casted)) - self.assertEqual(self.tf_dtype, casted.dtype) - self.assertTrue(self.tf_dtype, casted_val.dtype) + self.assertEqual(self.dtype, casted.dtype) + self.assertTrue(self.dtype, casted_val.dtype) def testUnknownRanksTTMatmul(self): # Tests tt_tt_matmul for matrices with unknown ranks - K_1 = tf.placeholder(self.tf_dtype, (1, 2, 2, None)) - K_2 = tf.placeholder(self.tf_dtype, (None, 3, 3, 1)) + K_1 = tf.placeholder(self.dtype, (1, 2, 2, None)) + K_2 = tf.placeholder(self.dtype, (None, 3, 3, 1)) tt_mat = TensorTrain([K_1, K_2]) res_actual = ops.full(ops.matmul(tt_mat, tt_mat)) res_desired = tf.matmul(ops.full(tt_mat), ops.full(tt_mat)) @@ -448,8 +448,8 @@ def testHalfKnownRanksTTMatmul(self): # Tests tt_tt_matmul for the case when one matrice has known ranks # and the other one doesn't np.random.seed(1) - K_1 = tf.placeholder(self.tf_dtype, (1, 2, 2, None)) - K_2 = tf.placeholder(self.tf_dtype, (None, 3, 3, 1)) + K_1 = tf.placeholder(self.dtype, (1, 2, 2, None)) + K_2 = tf.placeholder(self.dtype, (None, 3, 3, 1)) tt_mat_known_ranks = TensorTrain([K_1, K_2], tt_ranks=[1, 3, 1]) tt_mat = TensorTrain([K_1, K_2]) res_actual = ops.full(ops.matmul(tt_mat_known_ranks, tt_mat)) @@ -468,8 +468,8 @@ class _TTTensorBatchTest(): def testFullTensor2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(3, 10, rank).astype(self.tf_dtype.as_numpy_dtype) - b = np.random.rand(3, rank, 9).astype(self.tf_dtype.as_numpy_dtype) + a = np.random.rand(3, 10, rank).astype(self.dtype.as_numpy_dtype) + b = np.random.rand(3, rank, 9).astype(self.dtype.as_numpy_dtype) tt_cores = (a.reshape(3, 1, 10, rank), b.reshape(3, rank, 9, 1)) desired = np.einsum('oib,obj->oij', a, b) with self.test_session(): @@ -480,9 +480,9 @@ def testFullTensor2d(self): def testFullTensor3d(self): np.random.seed(1) for rank_1 in [1, 2]: - a = np.random.rand(3, 10, rank_1).astype(self.tf_dtype.as_numpy_dtype) - b = np.random.rand(3, rank_1, 9, 3).astype(self.tf_dtype.as_numpy_dtype) - c = np.random.rand(3, 3, 8).astype(self.tf_dtype.as_numpy_dtype) + a = np.random.rand(3, 10, rank_1).astype(self.dtype.as_numpy_dtype) + b = np.random.rand(3, rank_1, 9, 3).astype(self.dtype.as_numpy_dtype) + c = np.random.rand(3, 3, 8).astype(self.dtype.as_numpy_dtype) tt_cores = (a.reshape(3, 1, 10, rank_1), b, c.reshape((3, 3, 8, 1))) # Basically do full by hand. desired = np.einsum('oia,oajb,obk->oijk', a, b, c) @@ -501,10 +501,10 @@ def testFlatInnerTTTensbyTTTensSameBatchSize(self): for rank in rank_list: tt_1 = initializers.random_tensor_batch(shape, tt_rank=rank, batch_size=2, - dtype=self.tf_dtype) + dtype=self.dtype) tt_2 = initializers.random_tensor_batch(shape, tt_rank=rank, batch_size=2, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.flat_inner(tt_1, tt_2) tt_1_full = tf.reshape(ops.full(tt_1), (2, 1, -1)) tt_2_full = tf.reshape(ops.full(tt_2), (2, -1, 1)) @@ -515,9 +515,9 @@ def testFlatInnerTTTensbyTTTensSameBatchSize(self): def testFlatInnerTTTensbyTTTensBroadcasting(self): # Inner product between two batch TT-tensors with broadcasting. tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=1, - dtype=self.tf_dtype) + dtype=self.dtype) tt_2 = initializers.random_tensor_batch((2, 3, 4), batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual_1 = ops.flat_inner(tt_1, tt_2) res_actual_2 = ops.flat_inner(tt_2, tt_1) res_desired = tf.einsum('ijk,oijk->o', ops.full(tt_1[0]), ops.full(tt_2)) @@ -535,9 +535,9 @@ def testFlatInnerTTTensbyTTTensBroadcasting(self): def testAddSameBatchSize(self): # Sum two TT-tensors with the same batch size. tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) tt_b = initializers.random_tensor_batch((2, 1, 4), tt_rank=[1, 2, 4, 1], - batch_size=3, dtype=self.tf_dtype) + batch_size=3, dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_a + tt_b) @@ -550,9 +550,9 @@ def testAddSameBatchSize(self): def testAddBroadcasting(self): # Sum two TT-tensors with broadcasting. tt_a = initializers.random_tensor_batch((2, 1, 4), tt_rank=2, batch_size=1, - dtype=self.tf_dtype) + dtype=self.dtype) tt_b = initializers.random_tensor_batch((2, 1, 4), tt_rank=[1, 2, 4, 1], - batch_size=3, dtype=self.tf_dtype) + batch_size=3, dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_b + tt_a) @@ -565,7 +565,7 @@ def testAddBroadcasting(self): def testMultiplyByNumber(self): # Multiply batch of tensors by a number. tt = initializers.random_tensor_batch((1, 2, 3), tt_rank=(1, 2, 3, 1), - batch_size=3, dtype=self.tf_dtype) + batch_size=3, dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt, 4)) res_actual2 = ops.full(4.0 * tt) @@ -578,7 +578,7 @@ def testMultiplyByNumber(self): def testFrobeniusNormDifferentiableBatch(self): with self.test_session() as sess: tt = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) norm_sq_diff = ops.frobenius_norm_squared(tt, differentiable=True) variables = [norm_sq_diff, ops.full(tt)] norm_sq_diff_val, tt_full = sess.run(variables) @@ -589,7 +589,7 @@ def testFrobeniusNormTens(self): # Frobenius norm of a batch of TT-tensors. with self.test_session() as sess: tt = initializers.tensor_batch_with_random_cores((2, 1, 3), batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) norm_sq_actual = ops.frobenius_norm_squared(tt) norm_actual = ops.frobenius_norm(tt) vars = [norm_sq_actual, norm_actual, ops.full(tt)] @@ -602,9 +602,9 @@ def testFrobeniusNormTens(self): rtol=1e-5) def testMultiplyBatchByTensor(self): - tt_a = initializers.random_tensor((3, 3, 3), tt_rank=2, dtype=self.tf_dtype) + tt_a = initializers.random_tensor((3, 3, 3), tt_rank=2, dtype=self.dtype) tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt_a, tt_b)) res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) @@ -616,9 +616,9 @@ def testMultiplyBatchByTensor(self): def testMultiplyBatchByBatch(self): tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.full(ops.multiply(tt_a, tt_b)) res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) res_desired = ops.full(tt_a) * ops.full(tt_b) @@ -634,9 +634,9 @@ def testMultiplyBatchByBatch(self): def testMultiplyBroadcasting(self): tt_a = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=1, - dtype=self.tf_dtype) + dtype=self.dtype) tt_b = initializers.random_tensor_batch((3, 3, 3), tt_rank=2, batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.multiply(tt_a, tt_b)) res_actual2 = ops.full(ops.multiply(tt_b, tt_a)) @@ -647,13 +647,13 @@ def testMultiplyBroadcasting(self): self.assertAllClose(res_actual2_val, res_desired_val) def testMultiplyUnknownBatchSizeBroadcasting(self): - c1 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) - c2 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) + c1 = tf.placeholder(self.dtype, [None, 1, 3, 2]) + c2 = tf.placeholder(self.dtype, [None, 2, 3, 1]) tt_a = TensorTrainBatch([c1, c2]) tt_b = initializers.random_tensor_batch((3, 3), tt_rank=3, batch_size=1, - dtype=self.tf_dtype) + dtype=self.dtype) tt_c = initializers.random_tensor((3, 3), tt_rank=3, - dtype=self.tf_dtype) + dtype=self.dtype) res_ab = ops.full(ops.multiply(tt_a, tt_b)) res_ba = ops.full(ops.multiply(tt_b, tt_a)) res_ac = ops.full(ops.multiply(tt_a, tt_c)) @@ -671,10 +671,10 @@ def testMultiplyUnknownBatchSizeBroadcasting(self): self.assertAllClose(ca, des_ac) def testMultiplyTwoBatchesUnknownSize(self): - c1 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) - c2 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) - c3 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) - c4 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) + c1 = tf.placeholder(self.dtype, [None, 1, 3, 2]) + c2 = tf.placeholder(self.dtype, [None, 2, 3, 1]) + c3 = tf.placeholder(self.dtype, [None, 1, 3, 2]) + c4 = tf.placeholder(self.dtype, [None, 2, 3, 1]) tt_a = TensorTrainBatch([c1, c2]) tt_b = TensorTrainBatch([c3, c4]) res_ab = ops.full(ops.multiply(tt_a, tt_b)) @@ -699,10 +699,10 @@ def testMultiplyTwoBatchesUnknownSize(self): sess.run(to_run, feed_dict=feed_dict_err) def testMultiplyUnknownSizeBatchAndBatch(self): - c1 = tf.placeholder(self.tf_dtype, [None, 1, 3, 2]) - c2 = tf.placeholder(self.tf_dtype, [None, 2, 3, 1]) + c1 = tf.placeholder(self.dtype, [None, 1, 3, 2]) + c2 = tf.placeholder(self.dtype, [None, 2, 3, 1]) tt_b = initializers.random_tensor_batch((3, 3), tt_rank=2, batch_size=8, - dtype=self.tf_dtype) + dtype=self.dtype) tt_a = TensorTrainBatch([c1, c2]) res_ab = ops.full(ops.multiply(tt_a, tt_b)) res_ba = ops.full(ops.multiply(tt_b, tt_a)) @@ -724,7 +724,7 @@ def testMultiplyUnknownSizeBatchAndBatch(self): def testGatherND(self): idx = [[0, 0, 0], [0, 1, 2], [0, 1, 0]] pl_idx = tf.placeholder(tf.int32, [None, 3]) - tt = initializers.random_tensor((3, 4, 5), tt_rank=2, dtype=self.tf_dtype) + tt = initializers.random_tensor((3, 4, 5), tt_rank=2, dtype=self.dtype) res_np = ops.gather_nd(tt, idx) res_pl = ops.gather_nd(tt, pl_idx) res_desired = tf.gather_nd(ops.full(tt), idx) @@ -738,7 +738,7 @@ def testGatherNDBatch(self): idx = [[0, 0, 0, 0], [1, 0, 1, 2], [0, 0, 1, 0]] pl_idx = tf.placeholder(tf.int32, [None, 4]) tt = initializers.random_tensor_batch((3, 4, 5), tt_rank=2, batch_size=2, - dtype=self.tf_dtype) + dtype=self.dtype) res_np = ops.gather_nd(tt, idx) res_pl = ops.gather_nd(tt, pl_idx) res_desired = tf.gather_nd(ops.full(tt), idx) @@ -750,7 +750,7 @@ def testGatherNDBatch(self): def testCoreRenormBatch(self): a = initializers.random_tensor_batch(3 * (10,), tt_rank=7, batch_size=5, - dtype=self.tf_dtype) + dtype=self.dtype) b = ops.renormalize_tt_cores(a) var_list = [ops.full(a), ops.full(b)] @@ -769,8 +769,8 @@ class _TTMatrixTestBatch(): def testFullMatrix2d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(3, 2, 3, rank).astype(self.tf_dtype.as_numpy_dtype) - b = np.random.rand(3, rank, 4, 5).astype(self.tf_dtype.as_numpy_dtype) + a = np.random.rand(3, 2, 3, rank).astype(self.dtype.as_numpy_dtype) + b = np.random.rand(3, rank, 4, 5).astype(self.dtype.as_numpy_dtype) tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape((3, rank, 4, 5, 1))) # Basically do full by hand. desired = np.einsum('oijb,obkl->oijkl', a, b) @@ -785,9 +785,9 @@ def testFullMatrix2d(self): def testFullMatrix3d(self): np.random.seed(1) for rank in [1, 2]: - a = np.random.rand(3, 2, 3, rank).astype(self.tf_dtype.as_numpy_dtype) - b = np.random.rand(3, rank, 4, 5, rank).astype(self.tf_dtype.as_numpy_dtype) - c = np.random.rand(3, rank, 2, 2).astype(self.tf_dtype.as_numpy_dtype) + a = np.random.rand(3, 2, 3, rank).astype(self.dtype.as_numpy_dtype) + b = np.random.rand(3, rank, 4, 5, rank).astype(self.dtype.as_numpy_dtype) + c = np.random.rand(3, rank, 2, 2).astype(self.dtype.as_numpy_dtype) tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape(3, rank, 4, 5, rank), c.reshape(3, rank, 2, 2, 1)) # Basically do full by hand. @@ -809,10 +809,10 @@ def testTTMatTimesTTMatSameBatchSize(self): with self.test_session() as sess: tt_mat_1 = initializers.random_matrix_batch((left_shape, sum_shape), tt_rank=3, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) tt_mat_2 = initializers.random_matrix_batch((sum_shape, right_shape), batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) res_actual = ops.matmul(tt_mat_1, tt_mat_2) res_actual = ops.full(res_actual) res_desired = tf.matmul(ops.full(tt_mat_1), ops.full(tt_mat_2)) @@ -829,9 +829,9 @@ def testTTMatTimesTTMatBroadcasting(self): with self.test_session() as sess: tt_mat_1 = initializers.random_matrix_batch((left_shape, sum_shape), tt_rank=3, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) tt_mat_2 = initializers.random_matrix_batch((sum_shape, right_shape), - dtype=self.tf_dtype) + dtype=self.dtype) # TT-batch by one element TT-batch res_actual = ops.matmul(tt_mat_1, tt_mat_2) res_actual = ops.full(res_actual) @@ -850,7 +850,7 @@ def testTranspose(self): # Transpose a batch of TT-matrices. with self.test_session() as sess: tt = initializers.random_matrix_batch(((2, 3, 4), (2, 2, 2)), - batch_size=2, dtype=self.tf_dtype) + batch_size=2, dtype=self.dtype) res_actual = ops.full(ops.transpose(tt)) res_actual_val, tt_val = sess.run([res_actual, ops.full(tt)]) self.assertAllClose(tt_val.transpose((0, 2, 1)), res_actual_val) @@ -858,10 +858,10 @@ def testTranspose(self): def testAddSameBatchSize(self): # Sum two TT-matrices with the same batch size. tt_a = initializers.random_matrix_batch(((2, 1, 4), None), tt_rank=2, - batch_size=3, dtype=self.tf_dtype) + batch_size=3, dtype=self.dtype) tt_b = initializers.random_matrix_batch(((2, 1, 4), None), tt_rank=[1, 2, 4, 1], batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_a + tt_b) @@ -874,10 +874,10 @@ def testAddSameBatchSize(self): def testAddBroadcasting(self): # Sum two TT-matrices with broadcasting. tt_a = initializers.random_matrix_batch(((2, 1, 4), (2, 2, 2)), tt_rank=2, - batch_size=3, dtype=self.tf_dtype) + batch_size=3, dtype=self.dtype) tt_b = initializers.random_matrix_batch(((2, 1, 4), (2, 2, 2)), tt_rank=[1, 2, 4, 1], batch_size=1, - dtype=self.tf_dtype) + dtype=self.dtype) with self.test_session() as sess: res_actual = ops.full(ops.add(tt_a, tt_b)) res_actual2 = ops.full(tt_b + tt_a) @@ -893,10 +893,10 @@ def testCastFloat(self): batch_size=3) with self.test_session() as sess: - casted = ops.cast(tt_mat, self.tf_dtype) + casted = ops.cast(tt_mat, self.dtype) casted_val = sess.run(ops.full(casted)) - self.assertEqual(self.tf_dtype, casted.dtype) - self.assertTrue(self.tf_dtype, casted_val.dtype) + self.assertEqual(self.dtype, casted.dtype) + self.assertTrue(self.dtype, casted_val.dtype) def testCastIntFloat(self): # Tests cast function from int to float for matrices. @@ -908,52 +908,52 @@ def testCastIntFloat(self): tt_int_batch = shapes.expand_batch_dim(tt_int) with self.test_session() as sess: - casted = ops.cast(tt_int_batch, self.tf_dtype) + casted = ops.cast(tt_int_batch, self.dtype) casted_val = sess.run(ops.full(casted)) - self.assertEqual(self.tf_dtype, casted.dtype) - self.assertTrue(self.tf_dtype, casted_val.dtype) + self.assertEqual(self.dtype, casted.dtype) + self.assertTrue(self.dtype, casted_val.dtype) def _random_sparse(shape, non_zeros): sparse_flat_indices = np.random.choice(np.prod(shape), non_zeros).astype(int) sparse_indices = np.unravel_index(sparse_flat_indices, shape) sparse_indices = np.vstack(sparse_indices).transpose() - values = np.random.randn(non_zeros).astype(self.tf_dtype.as_numpy_dtype) + values = np.random.randn(non_zeros).astype(self.dtype.as_numpy_dtype) sparse = tf.SparseTensor(indices=sparse_indices, values=values, dense_shape=shape) return sparse class TTTensorTestFloat32(tf.test.TestCase, _TTTensorTest): - tf_dtype = tf.float32 + dtype = tf.float32 class TTTensorTestFloat64(tf.test.TestCase, _TTTensorTest): - tf_dtype = tf.float64 + dtype = tf.float64 class TTMatrixTestFloat32(tf.test.TestCase, _TTMatrixTest): - tf_dtype = tf.float32 + dtype = tf.float32 class TTMatrixTestFloat64(tf.test.TestCase, _TTMatrixTest): - tf_dtype = tf.float64 + dtype = tf.float64 class TTTensorBatchTestFloat32(tf.test.TestCase, _TTTensorBatchTest): - tf_dtype = tf.float32 + dtype = tf.float32 class TTTensorBatchTestFloat64(tf.test.TestCase, _TTTensorBatchTest): - tf_dtype = tf.float64 + dtype = tf.float64 class TTMatrixTestBatchFloat32(tf.test.TestCase, _TTMatrixTestBatch): - tf_dtype = tf.float32 + dtype = tf.float32 class TTMatrixTestBatchFloat64(tf.test.TestCase, _TTMatrixTestBatch): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": diff --git a/t3f/riemannian_test.py b/t3f/riemannian_test.py index 16752592..99d278d6 100644 --- a/t3f/riemannian_test.py +++ b/t3f/riemannian_test.py @@ -13,7 +13,7 @@ class _RiemannianTest(): def testProjectOnItself(self): # Projection of X into the tangent space of itself is X: P_x(x) = x. - tens = initializers.random_tensor((2, 3, 4), dtype=self.tf_dtype) + tens = initializers.random_tensor((2, 3, 4), dtype=self.dtype) proj = riemannian.project_sum(tens, tens) with self.test_session() as sess: actual_val, desired_val = sess.run((ops.full(proj), ops.full(tens))) @@ -32,7 +32,7 @@ def testProject(self): [[-0.19279465], [ 0.524976 ], [-0.40149197]]]) - convert = lambda t: np.array(t, dtype=self.tf_dtype.as_numpy_dtype) + convert = lambda t: np.array(t, dtype=self.dtype.as_numpy_dtype) tangent_tens_cores = list([convert(t) for t in tangent_tens_cores]) tangent_tens = TensorTrain(tangent_tens_cores, (4, 3), (1, 2, 1)) tens_cores = ([[[-1.01761142, 0.36075896, -0.2493624 ], @@ -60,14 +60,14 @@ def testProject(self): with self.test_session() as sess: proj_v = proj_full.eval() self.assertAllClose(desired_projection, proj_v) - self.assertEqual(self.tf_dtype.as_numpy_dtype, proj_v.dtype) + self.assertEqual(self.dtype.as_numpy_dtype, proj_v.dtype) def testProjectSum(self): # Test projecting a batch of TT-tensors. tens = initializers.random_tensor_batch((2, 3, 4), batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) tangent_tens = initializers.random_tensor((2, 3, 4), 3, - dtype=self.tf_dtype) + dtype=self.dtype) weighted_sum = tens[0] + tens[1] + tens[2] direct_proj = riemannian.project_sum(weighted_sum, tangent_tens) actual_proj = riemannian.project_sum(tens, tangent_tens) @@ -79,10 +79,10 @@ def testProjectSum(self): def testProjectWeightedSum(self): # Test projecting a batch of TT-tensors with providing coefs. tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4, - dtype=self.tf_dtype) + dtype=self.dtype) coef = [0.1, -2, 0, 0.4] tangent_tens = initializers.random_tensor((2, 3, 4), 4, - dtype=self.tf_dtype) + dtype=self.dtype) weighted_sum = coef[0] * tens[0] + coef[1] * tens[1] + coef[2] * tens[2] weighted_sum += coef[3] * tens[3] direct_proj = riemannian.project_sum(weighted_sum, tangent_tens) @@ -96,11 +96,11 @@ def testProjectWeightedSumMultipleOutputs(self): # Test projecting a batch of TT-tensors with providing weights and outputing # several TT objects with different weights. tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4, - dtype=self.tf_dtype) + dtype=self.dtype) np.random.seed(0) - weights = np.random.randn(4, 2).astype(self.tf_dtype.as_numpy_dtype) + weights = np.random.randn(4, 2).astype(self.dtype.as_numpy_dtype) tangent_tens = initializers.random_tensor((2, 3, 4), 4, - dtype=self.tf_dtype) + dtype=self.dtype) weighted_sum_1 = weights[0, 0] * tens[0] + weights[1, 0] * tens[1] +\ weights[2, 0] * tens[2] + weights[3, 0] * tens[3] weighted_sum_2 = weights[0, 1] * tens[0] + weights[1, 1] * tens[1] +\ @@ -120,7 +120,7 @@ def testProjectMatrixOnItself(self): # Project a TT-matrix on itself. # Projection of X into the tangent space of itself is X: P_x(x) = x. tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4)), - dtype=self.tf_dtype) + dtype=self.dtype) proj = riemannian.project_sum(tt_mat, tt_mat) with self.test_session() as sess: actual_val, desired_val = sess.run((ops.full(proj), ops.full(tt_mat))) @@ -129,9 +129,9 @@ def testProjectMatrixOnItself(self): def testCompareProjectSumAndProject(self): # Compare results of project_sum and project. tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4, - dtype=self.tf_dtype) + dtype=self.dtype) tangent_tens = initializers.random_tensor((2, 3, 4), 4, - dtype=self.tf_dtype) + dtype=self.dtype) project_sum = riemannian.project_sum(tens, tangent_tens, np.eye(4)) project = riemannian.project(tens, tangent_tens) with self.test_session() as sess: @@ -142,12 +142,12 @@ def testCompareProjectSumAndProject(self): def testProjectMatmul(self): # Project a TT-matrix times TT-vector on a TT-vector. tt_mat = initializers.random_matrix(((2, 3, 4), (2, 3, 4)), - dtype=self.tf_dtype) + dtype=self.dtype) tt_vec_what = initializers.random_matrix_batch(((2, 3, 4), None), batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) tt_vec_where = initializers.random_matrix(((2, 3, 4), None), - dtype=self.tf_dtype) + dtype=self.dtype) proj = riemannian.project_matmul(tt_vec_what, tt_vec_where, tt_mat) matvec = ops.matmul(tt_mat, tt_vec_what) proj_desired = riemannian.project(matvec, tt_vec_where) @@ -158,10 +158,10 @@ def testProjectMatmul(self): def testPairwiseFlatInnerTensor(self): # Compare pairwise_flat_inner_projected against naive implementation. what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) what2 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=4, - dtype=self.tf_dtype) - where = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) + dtype=self.dtype) + where = initializers.random_tensor((2, 3, 4), 3, dtype=self.dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) desired = batch_ops.pairwise_flat_inner(projected1, projected2) @@ -173,7 +173,7 @@ def testPairwiseFlatInnerTensor(self): with self.assertRaises(ValueError): # Second argument is not a projection on the tangent space. riemannian.pairwise_flat_inner_projected(projected1, what2) - where2 = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) + where2 = initializers.random_tensor((2, 3, 4), 3, dtype=self.dtype) another_projected2 = riemannian.project(what2, where2) with self.assertRaises(ValueError): # The arguments are projections on different tangent spaces. @@ -182,11 +182,11 @@ def testPairwiseFlatInnerTensor(self): def testPairwiseFlatInnerMatrix(self): # Compare pairwise_flat_inner_projected against naive implementation. what1 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) what2 = initializers.random_matrix_batch(((2, 3, 4), None), 4, batch_size=4, - dtype=self.tf_dtype) + dtype=self.dtype) where = initializers.random_matrix(((2, 3, 4), None), 3, - dtype=self.tf_dtype) + dtype=self.dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) desired = batch_ops.pairwise_flat_inner(projected1, projected2) @@ -199,7 +199,7 @@ def testPairwiseFlatInnerMatrix(self): # Second argument is not a projection on the tangent space. riemannian.pairwise_flat_inner_projected(projected1, what2) where2 = initializers.random_matrix(((2, 3, 4), None), 3, - dtype=self.tf_dtype) + dtype=self.dtype) another_projected2 = riemannian.project(what2, where2) with self.assertRaises(ValueError): # The arguments are projections on different tangent spaces. @@ -208,10 +208,10 @@ def testPairwiseFlatInnerMatrix(self): def testAddNProjected(self): # Add several TT-objects from the same tangent space. what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) what2 = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=3, - dtype=self.tf_dtype) - where = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) + dtype=self.dtype) + where = initializers.random_tensor((2, 3, 4), 3, dtype=self.dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) desired = ops.full(projected1 + projected2) @@ -223,7 +223,7 @@ def testAddNProjected(self): with self.assertRaises(ValueError): # Second argument is not a projection on the tangent space. riemannian.add_n_projected((projected1, what2)) - where2 = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) + where2 = initializers.random_tensor((2, 3, 4), 3, dtype=self.dtype) another_projected2 = riemannian.project(what2, where2) with self.assertRaises(ValueError): # The arguments are projections on different tangent spaces. @@ -231,9 +231,9 @@ def testAddNProjected(self): def testWeightedAddNProjected(self): # Add several TT-objects from the same tangent space with coefs. - what1 = initializers.random_tensor((2, 3, 4), 4, dtype=self.tf_dtype) - what2 = initializers.random_tensor((2, 3, 4), 1, dtype=self.tf_dtype) - where = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) + what1 = initializers.random_tensor((2, 3, 4), 4, dtype=self.dtype) + what2 = initializers.random_tensor((2, 3, 4), 1, dtype=self.dtype) + where = initializers.random_tensor((2, 3, 4), 3, dtype=self.dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) desired = ops.full(1.2 * projected1 + -2.0 * projected2) @@ -246,7 +246,7 @@ def testWeightedAddNProjected(self): with self.assertRaises(ValueError): # Second argument is not a projection on the tangent space. riemannian.add_n_projected((projected1, what2), coef=[1.2, -2.0]) - where2 = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) + where2 = initializers.random_tensor((2, 3, 4), 3, dtype=self.dtype) another_projected2 = riemannian.project(what2, where2) with self.assertRaises(ValueError): # The arguments are projections on different tangent spaces. @@ -256,10 +256,10 @@ def testWeightedAddNProjected(self): def testWeightedAddNProjectedBatch(self): # Add several TT-batches from the same tangent space with coefs. what1 = initializers.random_tensor_batch((2, 3, 4), 4, batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) what2 = initializers.random_tensor_batch((2, 3, 4), 1, batch_size=3, - dtype=self.tf_dtype) - where = initializers.random_tensor((2, 3, 4), 3, dtype=self.tf_dtype) + dtype=self.dtype) + where = initializers.random_tensor((2, 3, 4), 3, dtype=self.dtype) projected1 = riemannian.project(what1, where) projected2 = riemannian.project(what2, where) @@ -276,11 +276,11 @@ def testWeightedAddNProjectedBatch(self): class RiemannianTestFloat32(tf.test.TestCase, _RiemannianTest): - tf_dtype = tf.float32 + dtype = tf.float32 class RiemannianTestFloat64(tf.test.TestCase, _RiemannianTest): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": diff --git a/t3f/tensor_train_batch_test.py b/t3f/tensor_train_batch_test.py index 78bb8bd0..2fa43c21 100644 --- a/t3f/tensor_train_batch_test.py +++ b/t3f/tensor_train_batch_test.py @@ -9,7 +9,7 @@ class _TensorTrainBatchTest(): def testTensorIndexing(self): tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) with self.test_session() as sess: desired = ops.full(tens)[:, :, :, :] actual = ops.full(tens[:, :, :, :]) @@ -50,7 +50,7 @@ def testTensorIndexing(self): def testPlaceholderTensorIndexing(self): tens = initializers.random_tensor_batch((3, 3, 4), batch_size=3, - dtype=self.tf_dtype) + dtype=self.dtype) with self.test_session() as sess: start = tf.placeholder(tf.int32) end = tf.placeholder(tf.int32) @@ -78,17 +78,17 @@ def testPlaceholderTensorIndexing(self): def testShapeOverflow(self): large_shape = [10] * 20 tensor = initializers.random_matrix_batch([large_shape, large_shape], - batch_size=5, dtype=self.tf_dtype) + batch_size=5, dtype=self.dtype) shape = tensor.get_shape() self.assertEqual([5, 10 ** 20, 10 ** 20], shape) class TensorTrainBatchTestFloat32(tf.test.TestCase, _TensorTrainBatchTest): - tf_dtype = tf.float32 + dtype = tf.float32 class TensorTrainBatchTestFloat64(tf.test.TestCase, _TensorTrainBatchTest): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": diff --git a/t3f/tensor_train_test.py b/t3f/tensor_train_test.py index e7b37fc0..cb18a6ad 100644 --- a/t3f/tensor_train_test.py +++ b/t3f/tensor_train_test.py @@ -27,8 +27,8 @@ def testValidateTTCores2d(self): ((1, 2, 1, 1), (1, 2, 1), False)) for tt_ranks, claimed_tt_ranks, desired in schedule: - a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1]), dtype=self.tf_dtype) - b = tf.random_normal((tt_ranks[2], 9, tt_ranks[3]), dtype=self.tf_dtype) + a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1]), dtype=self.dtype) + b = tf.random_normal((tt_ranks[2], 9, tt_ranks[3]), dtype=self.dtype) with self.test_session(): actual = tensor_train._are_tt_cores_valid((a, b), (10, 9), claimed_tt_ranks) @@ -72,9 +72,9 @@ def testValidateTTCores3d(self): ((1, 2, 2, 3, 3, 1), None, True)) for tt_ranks, claimed_tt_ranks, desired in schedule: - a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1]), dtype=self.tf_dtype) - b = tf.random_normal((tt_ranks[2], 1, tt_ranks[3]), dtype=self.tf_dtype) - c = tf.random_normal((tt_ranks[4], 2, tt_ranks[5]), dtype=self.tf_dtype) + a = tf.random_normal((tt_ranks[0], 10, tt_ranks[1]), dtype=self.dtype) + b = tf.random_normal((tt_ranks[2], 1, tt_ranks[3]), dtype=self.dtype) + c = tf.random_normal((tt_ranks[4], 2, tt_ranks[5]), dtype=self.dtype) with self.test_session(): actual = tensor_train._are_tt_cores_valid((a, b, c), (10, 1, 2), claimed_tt_ranks) @@ -96,7 +96,7 @@ def testValidateTTCores3d(self): tensor_train.TensorTrain((a, b_new, c), (10, 1, 2), claimed_tt_ranks) def testTensorIndexing(self): - tens = initializers.random_tensor((3, 3, 4), dtype=self.tf_dtype) + tens = initializers.random_tensor((3, 3, 4), dtype=self.dtype) with self.test_session() as sess: desired = ops.full(tens)[:, :, :] actual = ops.full(tens[:, :, :]) @@ -126,7 +126,7 @@ def testTensorIndexing(self): tens[1, 1] def testPlaceholderTensorIndexing(self): - tens = initializers.random_tensor((3, 3, 4), dtype=self.tf_dtype) + tens = initializers.random_tensor((3, 3, 4), dtype=self.dtype) with self.test_session() as sess: start = tf.placeholder(tf.int32) end = tf.placeholder(tf.int32) @@ -138,17 +138,17 @@ def testPlaceholderTensorIndexing(self): def testShapeOverflow(self): large_shape = [10] * 20 matrix = initializers.matrix_zeros([large_shape, large_shape], - dtype=self.tf_dtype) + dtype=self.dtype) shape = matrix.get_shape() self.assertEqual([10 ** 20, 10 ** 20], shape) class TensorTrainTestFloat32(tf.test.TestCase, _TensorTrainTest): - tf_dtype = tf.float32 + dtype = tf.float32 class TensorTrainTestFloat64(tf.test.TestCase, _TensorTrainTest): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": diff --git a/t3f/variables_test.py b/t3f/variables_test.py index aec65b71..7b36d2b6 100644 --- a/t3f/variables_test.py +++ b/t3f/variables_test.py @@ -9,7 +9,7 @@ class _VariablesTest(): def testGetExistingVariable(self): - init = initializers.random_tensor([2, 3, 2], tt_rank=2, dtype=self.tf_dtype) + init = initializers.random_tensor([2, 3, 2], tt_rank=2, dtype=self.dtype) tt_1 = variables.get_variable('tt_1', initializer=init) with tf.variable_scope('test'): tt_2 = variables.get_variable('tt_2', initializer=init) @@ -24,14 +24,14 @@ def testGetExistingVariable(self): variables.get_variable('tt_3') with tf.variable_scope('', reuse=True): - tt_1_copy = variables.get_variable('tt_1', dtype=self.tf_dtype) + tt_1_copy = variables.get_variable('tt_1', dtype=self.dtype) self.assertAllClose(ops.full(tt_1).eval(), ops.full(tt_1_copy).eval()) with tf.variable_scope('', reuse=True): # Again try to retrieve an existing variable, but pass an initializer # and check that it still works. tt_1_copy = variables.get_variable('tt_1', initializer=0 * init, - dtype=self.tf_dtype) + dtype=self.dtype) self.assertAllClose(ops.full(tt_1).eval(), ops.full(tt_1_copy).eval()) with self.assertRaises(ValueError): @@ -45,16 +45,16 @@ def testGetExistingVariable(self): variables.get_variable('tt_2') with tf.variable_scope('test', reuse=True): - tt_2_copy = variables.get_variable('tt_2', dtype=self.tf_dtype) + tt_2_copy = variables.get_variable('tt_2', dtype=self.dtype) self.assertAllClose(ops.full(tt_2).eval(), ops.full(tt_2_copy).eval()) def testAttributes(self): # Test that after converting an initializer into a variable all the # attributes stays the same. - tens = initializers.random_tensor([2, 3, 2], tt_rank=2, dtype=self.tf_dtype) + tens = initializers.random_tensor([2, 3, 2], tt_rank=2, dtype=self.dtype) tens_v = variables.get_variable('tt_tens', initializer=tens) mat = initializers.random_matrix([[3, 2, 2], [3, 3, 3]], tt_rank=3, - dtype=self.tf_dtype) + dtype=self.dtype) mat_v = variables.get_variable('tt_mat', initializer=mat) for (init, var) in [[tens, tens_v], [mat, mat_v]]: self.assertEqual(init.get_shape(), var.get_shape()) @@ -65,10 +65,10 @@ def testAttributes(self): def testAssign(self): old_init = initializers.random_tensor([2, 3, 2], tt_rank=2, - dtype=self.tf_dtype) + dtype=self.dtype) tt = variables.get_variable('tt', initializer=old_init) new_init = initializers.random_tensor([2, 3, 2], tt_rank=2, - dtype=self.tf_dtype) + dtype=self.dtype) assigner = variables.assign(tt, new_init) with self.test_session(): tf.global_variables_initializer().run() @@ -84,11 +84,11 @@ def testAssign(self): class VariablesTestFloat32(tf.test.TestCase, _VariablesTest): - tf_dtype = tf.float32 + dtype = tf.float32 class VariablesTestFloat64(tf.test.TestCase, _VariablesTest): - tf_dtype = tf.float64 + dtype = tf.float64 if __name__ == "__main__": From 6f204dcf49aa58b67850769013da96a350e88b19 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 20:40:37 +0000 Subject: [PATCH 093/233] not very important forgotten dtype set in tests --- t3f/ops_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t3f/ops_test.py b/t3f/ops_test.py index 51c21900..964262d1 100644 --- a/t3f/ops_test.py +++ b/t3f/ops_test.py @@ -527,7 +527,8 @@ def testFlatInnerTTTensbyTTTensBroadcasting(self): self.assertAllClose(res_actual_1_val, res_desired_val) self.assertAllClose(res_actual_2_val, res_desired_val) - tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=2) + tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=2, + dtype=self.dtype) with self.assertRaises(ValueError): # The batch_sizes are different. ops.flat_inner(tt_1, tt_2) From 4cf23554fce9ce9a5b7c9b98503df0e2ea58836d Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 20:40:52 +0000 Subject: [PATCH 094/233] shapes: test with different dtypes --- t3f/shapes_test.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/t3f/shapes_test.py b/t3f/shapes_test.py index bf0decaf..df2f7586 100644 --- a/t3f/shapes_test.py +++ b/t3f/shapes_test.py @@ -4,13 +4,22 @@ from t3f import shapes -class ShapesTest(tf.test.TestCase): +class _ShapesTest(): def testLazyShapeOverflow(self): large_shape = [10] * 20 - tensor = initializers.random_matrix_batch([large_shape, large_shape], batch_size=5) + tensor = initializers.random_matrix_batch([large_shape, large_shape], + batch_size=5, dtype=self.dtype) self.assertAllEqual([5, 10 ** 20, 10 ** 20], shapes.lazy_shape(tensor)) +class ShapesTestFloat32(tf.test.TestCase, _ShapesTest): + dtype = tf.float32 + + +class ShapesTestFloat64(tf.test.TestCase, _ShapesTest): + dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From 15e6dae4c8af709e85f5ca0f8e2a5478ca4dc06c Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 20:43:49 +0000 Subject: [PATCH 095/233] remove redundant dtype convertions --- t3f/approximate_test.py | 2 +- t3f/riemannian_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/approximate_test.py b/t3f/approximate_test.py index 9082bf20..91e23a65 100644 --- a/t3f/approximate_test.py +++ b/t3f/approximate_test.py @@ -90,7 +90,7 @@ def desired(tt_batch, coef): coef = [[1., 0.1], [0.9, -0.2], [0.3, 0.3]] - coef = np.array(coef).astype(self.dtype.as_numpy_dtype) + coef = np.array(coef) res_actual = ops.full(approximate.reduce_sum_batch(tt_batch, 6, coef)) res_desired_1 = ops.full(desired(tt_batch, coef[:, 0])) diff --git a/t3f/riemannian_test.py b/t3f/riemannian_test.py index 99d278d6..3c808190 100644 --- a/t3f/riemannian_test.py +++ b/t3f/riemannian_test.py @@ -98,7 +98,7 @@ def testProjectWeightedSumMultipleOutputs(self): tens = initializers.random_tensor_batch((2, 3, 4), 3, batch_size=4, dtype=self.dtype) np.random.seed(0) - weights = np.random.randn(4, 2).astype(self.dtype.as_numpy_dtype) + weights = np.random.randn(4, 2) tangent_tens = initializers.random_tensor((2, 3, 4), 4, dtype=self.dtype) weighted_sum_1 = weights[0, 0] * tens[0] + weights[1, 0] * tens[1] +\ From 6dd8aca31accc718076c612f234a744d6a4fe458 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 21:17:24 +0000 Subject: [PATCH 096/233] check dtype if epsilon is tf.tensor (and not python float) --- t3f/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/ops.py b/t3f/ops.py index 97ec63b9..27f5216d 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -1179,7 +1179,7 @@ def renormalize_tt_cores(tt, epsilon=1e-8): case applies to each TT in `TensorTrainBatch`. """ - epsilon = tf.cast(epsilon, tt.dtype) + epsilon = tf.convert_to_tensor(epsilon, dtype=tt.dtype) if isinstance(tt, TensorTrain): new_cores = [] running_log_norm = 0 From 9035216739a88004f66134e84fc02b8d038b69e9 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 21:33:51 +0000 Subject: [PATCH 097/233] Remove accidental parenthesis --- t3f/riemannian_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/riemannian_test.py b/t3f/riemannian_test.py index 3c808190..df3f5246 100644 --- a/t3f/riemannian_test.py +++ b/t3f/riemannian_test.py @@ -50,7 +50,7 @@ def testProject(self): [ 0.6577514 ], [ 2.13703185]]]) tens_cores = list([convert(t) for t in tens_cores]) - tens = TensorTrain((tens_cores), (4, 3), (1, 3, 1)) + tens = TensorTrain(tens_cores, (4, 3), (1, 3, 1)) desired_projection = [[-0.67638254, -1.17163914, 0.29850939], [-1.66479093, -0.99003251, 2.46629195], [-0.04847773, -0.72908174, 0.20142675], From 43dc9eb03eaf24ae222b2af3e6df90902051f35b Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 21:44:22 +0000 Subject: [PATCH 098/233] remove draft jupyter notebook --- t3f/Auto diff.ipynb | 515 -------------------------------------------- 1 file changed, 515 deletions(-) delete mode 100644 t3f/Auto diff.ipynb diff --git a/t3f/Auto diff.ipynb b/t3f/Auto diff.ipynb deleted file mode 100644 index c39acd17..00000000 --- a/t3f/Auto diff.ipynb +++ /dev/null @@ -1,515 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import t3f\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# tf.enable_eager_execution()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "d = 3\n", - "w = t3f.random_matrix(([10] * d, None))\n", - "w = t3f.get_variable('w', initializer=w)\n", - "A = t3f.random_matrix(([10] * d, [10] * d))\n", - "A = t3f.get_variable('A', initializer=A)\n", - "z = t3f.random_matrix(([10] * d, None))\n", - "z = t3f.get_variable('z', initializer=z)\n", - "x = t3f.random_matrix(([10] * d, None), tt_rank=100)\n", - "x = t3f.get_variable('x', initializer=x)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "w.is_tt_matrix()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "def tangent_space_to_deltas(tt):\n", - " if tt.projection_on is None:\n", - " raise ValueError('tt argument is supposed to be a projection, but it lacks projection_on field')\n", - " num_dims = tt.ndims()\n", - " deltas = [None] * num_dims\n", - " if tt.is_tt_matrix():\n", - " for i in range(1, num_dims - 1):\n", - " r1, _, _, r2 = tt.tt_cores[i].get_shape().as_list()\n", - " if int(r1 / 2) != r1 / 2:\n", - " raise ValueError('tt argument is supposed to be a projection, but its ranks are not even.')\n", - " deltas[i] = tt.tt_cores[i][int(r1 / 2):, :, :, :int(r2 / 2)]\n", - " _, _, _, r = tt.tt_cores[0].get_shape().as_list()\n", - " deltas[0] = tt.tt_cores[0][:, :, :, :int(r / 2)]\n", - " r, _, _, _ = tt.tt_cores[num_dims - 1].get_shape().as_list()\n", - " deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][int(r / 2):, :, :, :]\n", - " else:\n", - " for i in range(1, num_dims - 1):\n", - " r1, _, r2 = tt.tt_cores[i].get_shape().as_list()\n", - " if int(r1 / 2) != r1 / 2:\n", - " raise ValueError('tt argument is supposed to be a projection, but its ranks are not even.')\n", - " deltas[i] = tt.tt_cores[i][int(r1 / 2):, :, :int(r2 / 2)]\n", - " _, _, r = tt.tt_cores[0].get_shape().as_list()\n", - " deltas[0] = tt.tt_cores[0][:, :, :int(r / 2)]\n", - " r, _, _ = tt.tt_cores[num_dims - 1].get_shape().as_list()\n", - " deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][int(r / 2):, :, :]\n", - " return deltas\n", - "\n", - "def left_q(X, i):\n", - " \"\"\"Compute the orthogonal matrix Q_{\\leq i} as defined in [1].\"\"\"\n", - " if i < 0:\n", - " return np.ones([1, 1], dtype=np.float32)\n", - " answ = np.ones([1, 1])\n", - " for dim in range(i + 1):\n", - " answ = np.tensordot(answ, sess.run(X.tt_cores[dim]), 1)\n", - " answ = np.reshape(answ, (-1, answ.shape[-1]))\n", - " return answ.astype(np.float32)\n", - "\n", - "def right_q(X, i):\n", - " \"\"\"Compute the orthogonal matrix Q_{\\geq i} as defined in [1].\"\"\"\n", - " if i > X.ndims() - 1:\n", - " return np.ones([1, 1], dtype=np.float32)\n", - " answ = np.ones([1, 1])\n", - " for dim in range(X.ndims() - 1, i - 1, -1):\n", - " answ = np.tensordot(sess.run(X.tt_cores[dim]), answ, 1)\n", - " answ = np.reshape(answ, (answ.shape[0], -1))\n", - " return answ.T.astype(np.float32)\n", - "\n", - "def deltas_to_tangent_space(deltas, tt, left, right):\n", - " cores = []\n", - " dtype = deltas[0].dtype\n", - " num_dims = left.ndims()\n", - " left_tangent_tt_ranks = t3f.shapes.lazy_tt_ranks(left)\n", - " right_tangent_tt_ranks = t3f.shapes.lazy_tt_ranks(left)\n", - " raw_shape = t3f.shapes.lazy_raw_shape(left)\n", - " right_rank_dim = left.right_tt_rank_dim\n", - " left_rank_dim = left.left_tt_rank_dim\n", - " for i in range(num_dims):\n", - " left_tt_core = left.tt_cores[i]\n", - " right_tt_core = right.tt_cores[i]\n", - "\n", - " if i == 0:\n", - " tangent_core = tf.concat((deltas[i], left_tt_core),\n", - " axis=right_rank_dim)\n", - " elif i == num_dims - 1:\n", - " tangent_core = tf.concat((right_tt_core, deltas[i]),\n", - " axis=left_rank_dim)\n", - " else:\n", - " rank_1 = right_tangent_tt_ranks[i]\n", - " rank_2 = left_tangent_tt_ranks[i + 1]\n", - " if tt.is_tt_matrix():\n", - " mode_size_n = raw_shape[0][i]\n", - " mode_size_m = raw_shape[1][i]\n", - " shape = [rank_1, mode_size_n, mode_size_m, rank_2]\n", - " else:\n", - " mode_size_n = raw_shape[0][i]\n", - " shape = [rank_1, mode_size_n, rank_2]\n", - " zeros = tf.zeros(shape, dtype)\n", - " upper = tf.concat((right_tt_core, zeros), axis=right_rank_dim)\n", - " lower = tf.concat((deltas[i], left_tt_core), axis=right_rank_dim)\n", - " tangent_core = tf.concat((upper, lower), axis=left_rank_dim)\n", - " cores.append(tangent_core)\n", - " tangent = t3f.TensorTrain(cores)\n", - " tangent.projection_on = tt\n", - " return tangent\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 2])" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.array([[1, 2], [3, 4]])[0, :]" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 2, 3, 4])" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.array([[1, 2], [3, 4]]).flatten()" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [], - "source": [ - "def _riemannian_grad(func, w, w_projection, left, right):\n", - " h = func(w_projection)\n", - " cores_grad = tf.gradients(h, w_projection.tt_cores)\n", - " deltas = []\n", - " for i in range(w.ndims()):\n", - " if w.is_tt_matrix():\n", - " r1, n, m, r2 = left.tt_cores[i].shape.as_list()\n", - " else:\n", - " r1, n, r2 = left.tt_cores[i].shape.as_list()\n", - " q = tf.reshape(left.tt_cores[i], (-1, r2))\n", - " if w.is_tt_matrix():\n", - " if i == 0:\n", - " curr_grad = cores_grad[i][:, :, :, :r2]\n", - " elif i == w.ndims() - 1:\n", - " curr_grad = cores_grad[i][r1:, :, :, :]\n", - " else:\n", - " curr_grad = cores_grad[i][r1:, :, :, :r2]\n", - " else:\n", - " if i == 0:\n", - " curr_grad = cores_grad[i][:, :, :r2]\n", - " elif i == w.ndims() - 1:\n", - " curr_grad = cores_grad[i][r1:, :, :]\n", - " else:\n", - " curr_grad = cores_grad[i][r1:, :, :r2]\n", - " if i < w.ndims() - 1:\n", - " proj = (tf.eye(r1 * n) - q @ tf.transpose(q))\n", - " # TODO: multiply faster.\n", - " delta = proj @ tf.reshape(curr_grad, (-1, r2))\n", - " delta = tf.reshape(delta, left.tt_cores[i].shape)\n", - " else:\n", - " delta = curr_grad\n", - " deltas.append(delta)\n", - " return deltas_to_tangent_space(deltas, w, left, right)\n", - "\n", - "def riemannian_grad(func, w):\n", - " left = t3f.orthogonalize_tt_cores(w)\n", - " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", - " deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]]\n", - " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", - " return _riemannian_grad(func, w, w_projection, left, right)\n", - "\n", - "def hessian_by_vector(f, w, vector):\n", - " left = t3f.orthogonalize_tt_cores(w)\n", - " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", - " vector_projected = t3f.project(vector, w)\n", - " vector_projected = t3f.expand_batch_dim(vector_projected)\n", - " vector_projected.projection_on = w\n", - " def new_f(new_w):\n", - " grad = _riemannian_grad(f, w, new_w, left, right)\n", - " grad = t3f.expand_batch_dim(grad)\n", - " # TODO: durty hack.\n", - " grad.projection_on = w\n", - " return t3f.pairwise_flat_inner_projected(grad, vector_projected)[0, 0]\n", - " return riemannian_grad(new_f, w)\n", - "\n", - "def block_diag_hessian_by_vector(f, w, vector):\n", - " left = t3f.orthogonalize_tt_cores(w)\n", - " right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", - " original_zeros = []\n", - " deltas = []\n", - " for i in range(w.ndims()):\n", - " shape = right.tt_cores[i].get_shape().as_list()\n", - " shape += [w.ndims()]\n", - " curr_original_zero = tf.zeros(shape)\n", - " original_zeros.append(curr_original_zero)\n", - " delta = tf.reduce_sum(curr_original_zero, axis=-1)\n", - " deltas.append(delta)\n", - " deltas[0] += right.tt_cores[0]\n", - " w_projection = deltas_to_tangent_space(deltas, w, left, right)\n", - "# grad = _riemannian_grad(f, w, w_projection, left, right)\n", - " vector_projected = t3f.project(vector, w)\n", - " vector_deltas = tangent_space_to_deltas(vector_projected)\n", - "# grad_deltas = tangent_space_to_deltas(grad)\n", - " h = f(w_projection)\n", - " cores_grad = tf.gradients(h, original_zeros)\n", - " grad_deltas = []\n", - " for i in range(w.ndims()):\n", - " if w.is_tt_matrix():\n", - " r1, n, m, r2 = left.tt_cores[i].shape.as_list()\n", - " else:\n", - " r1, n, r2 = left.tt_cores[i].shape.as_list()\n", - " q = tf.reshape(left.tt_cores[i], (-1, r2))\n", - " if w.is_tt_matrix():\n", - " curr_grad = cores_grad[i][:, :, :, :, i]\n", - " else:\n", - " curr_grad = cores_grad[i][:, :, :, i]\n", - " if i < w.ndims() - 1:\n", - " proj = (tf.eye(r1 * n) - q @ tf.transpose(q))\n", - " # TODO: multiply faster.\n", - " delta = proj @ tf.reshape(curr_grad, (-1, r2))\n", - " delta = tf.reshape(delta, left.tt_cores[i].shape)\n", - " else:\n", - " delta = curr_grad\n", - " grad_deltas.append(delta)\n", - " \n", - " intermidiate_f = 0.0\n", - " for i in range(w.ndims()):\n", - " intermidiate_f += tf.reduce_sum(grad_deltas[i] * vector_deltas[i])\n", - " final_deltas = []\n", - " cores_grad = tf.gradients(intermidiate_f, original_zeros)\n", - " for i in range(w.ndims()):\n", - " if w.is_tt_matrix():\n", - " r1, n, m, r2 = left.tt_cores[i].shape.as_list()\n", - " else:\n", - " r1, n, r2 = left.tt_cores[i].shape.as_list()\n", - " q = tf.reshape(left.tt_cores[i], (-1, r2))\n", - " if w.is_tt_matrix():\n", - " curr_grad = cores_grad[i][:, :, :, :, i]\n", - " else:\n", - " curr_grad = cores_grad[i][:, :, :, i]\n", - "# import pdb; pdb.set_trace()\n", - " if i < w.ndims() - 1:\n", - " proj = (tf.eye(r1 * n) - q @ tf.transpose(q))\n", - " # TODO: multiply faster.\n", - " delta = proj @ tf.reshape(curr_grad, (-1, r2))\n", - " delta = tf.reshape(delta, left.tt_cores[i].shape)\n", - " else:\n", - " delta = curr_grad\n", - " final_deltas.append(delta)\n", - " return deltas_to_tangent_space(final_deltas, w, left, right)\n", - " \n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [], - "source": [ - "# Try block diag hessian\n", - "deltas = tangent_space_to_deltas(t3f.project(z, w))\n", - "deltas[1:] = [tf.zeros_like(d) for d in deltas[1:]]\n", - "left = t3f.orthogonalize_tt_cores(w)\n", - "right = t3f.orthogonalize_tt_cores(left, left_to_right=False)\n", - "new_z = deltas_to_tangent_space(deltas, w, left, right)\n", - "block_diag = hessian_by_vector(func, w, new_z)\n", - "actual = hessian_by_vector(func, w, z)" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "ename": "AssertionError", - "evalue": "\nNot equal to tolerance rtol=0.0001, atol=0\n\n(mismatch 100.0%)\n x: array([[[[14.918402, -4.146306]],\n\n [[ 1.740906, -5.047251]],...\n y: array([[[[ 19.173162, -6.58578 ]],\n\n [[ -2.820656, -1.763076]],...", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAssertionError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[0mdesired\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtangent_space_to_deltas\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhessian_by_vector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_z\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 6\u001b[0m \u001b[1;31m# print(desired, tangent_space_to_deltas(block_diag_hessian_by_vector(func, w, z)))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtesting\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0massert_allclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdesired\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtangent_space_to_deltas\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mblock_diag_hessian_by_vector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mz\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrtol\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1e-4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;32m~\\AppData\\Local\\Continuum\\miniconda3\\lib\\site-packages\\numpy\\testing\\nose_tools\\utils.py\u001b[0m in \u001b[0;36massert_allclose\u001b[1;34m(actual, desired, rtol, atol, equal_nan, err_msg, verbose)\u001b[0m\n\u001b[0;32m 1394\u001b[0m \u001b[0mheader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'Not equal to tolerance rtol=%g, atol=%g'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mrtol\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0matol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1395\u001b[0m assert_array_compare(compare, actual, desired, err_msg=str(err_msg),\n\u001b[1;32m-> 1396\u001b[1;33m verbose=verbose, header=header, equal_nan=equal_nan)\n\u001b[0m\u001b[0;32m 1397\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1398\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\AppData\\Local\\Continuum\\miniconda3\\lib\\site-packages\\numpy\\testing\\nose_tools\\utils.py\u001b[0m in \u001b[0;36massert_array_compare\u001b[1;34m(comparison, x, y, err_msg, verbose, header, precision, equal_nan, equal_inf)\u001b[0m\n\u001b[0;32m 777\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mheader\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 778\u001b[0m names=('x', 'y'), precision=precision)\n\u001b[1;32m--> 779\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mAssertionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 780\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 781\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtraceback\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mAssertionError\u001b[0m: \nNot equal to tolerance rtol=0.0001, atol=0\n\n(mismatch 100.0%)\n x: array([[[[14.918402, -4.146306]],\n\n [[ 1.740906, -5.047251]],...\n y: array([[[[ 19.173162, -6.58578 ]],\n\n [[ -2.820656, -1.763076]],..." - ] - } - ], - "source": [ - "for i in range(w.ndims()):\n", - " deltas = [tf.zeros_like(d) for d in tangent_space_to_deltas(t3f.project(z, w))]\n", - " deltas[i] = tangent_space_to_deltas(t3f.project(z, w))[i]\n", - " new_z = deltas_to_tangent_space(deltas, w, left, right)\n", - " desired = tangent_space_to_deltas(hessian_by_vector(func, w, new_z))[i]\n", - "# print(desired, tangent_space_to_deltas(block_diag_hessian_by_vector(func, w, z)))\n", - " np.testing.assert_allclose(*sess.run([desired, tangent_space_to_deltas(block_diag_hessian_by_vector(func, w, z))[i]]), rtol=1e-4)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "sess = tf.Session()\n", - "sess.run(tf.global_variables_initializer())" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "func = lambda w: 0.5 * t3f.flat_inner(x, w)**2\n", - "desired, actual = sess.run([t3f.full(t3f.flat_inner(x, w) * t3f.project(x, w)), t3f.full(riemannian_grad(func, w))])\n", - "np.testing.assert_allclose(desired, actual, rtol=1e-3)\n", - "\n", - "func = lambda w: t3f.quadratic_form(A, w, w)\n", - "desired = t3f.project(t3f.matmul(t3f.transpose(A) + A, t3f.project(z, w)), w)\n", - "actual = hessian_by_vector(func, w, z)\n", - "desired, actual = sess.run([t3f.full(desired), t3f.full(actual)])\n", - "np.testing.assert_allclose(desired, actual, rtol=1e-2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "op1 = riemannian_grad(func, w).op\n", - "op2 = (t3f.project_matmul(t3f.expand_batch_dim(w), w, A)).op" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%timeit sess.run(op1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%timeit sess.run(op2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "t1 = tf.zeros((3, 4))\n", - "t2 = tf.ones((3, 4))\n", - "h1 = tf.reduce_sum(t1 * t2)\n", - "h2 = tf.reduce_prod(t1 * t2)\n", - "tf.gradients([h1, h2], [t1, t2])" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "op1 = hessian_by_vector(func, w, z).op\n", - "op2 = block_diag_hessian_by_vector(func, w, z).op" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "225 µs ± 59.9 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "258 µs ± 28.6 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n", - "194 µs ± 23 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n", - "229 µs ± 44.5 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n" - ] - } - ], - "source": [ - "\n", - "%timeit sess.run(op1)\n", - "%timeit sess.run(op2)\n", - "\n", - "%timeit sess.run(op1)\n", - "%timeit sess.run(op2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From d904a375a5704983d11cc863a3832f33c5ed6cc4 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 21:57:03 +0000 Subject: [PATCH 099/233] Another bug fix in hessian by vector: in desired use projected z --- t3f/autodiff_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index 0ee371cb..111a2517 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -49,7 +49,8 @@ def func1(x): # Hessian by vector: w actual1 = ops.full(autodiff.hessian_vector_product(func1, x, z)) - desired1 = riemannian.project(ops.flat_inner(z, w) * w, x) + projected_z = riemannian.project(z, x) + desired1 = riemannian.project(ops.flat_inner(projected_z, w) * w, x) desired1 = ops.full(desired1) with self.test_session() as sess: actual1_v, desired1_v = sess.run([actual1, desired1]) From b6909570a71d4506ba901bd35d80257e625b61a6 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 22:04:05 +0000 Subject: [PATCH 100/233] New dtype style testing --- t3f/autodiff_test.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index 111a2517..2287dfa0 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -8,13 +8,13 @@ from t3f import autodiff -class AutodiffTest(tf.test.TestCase): +class _AutodiffTest(): def testGradients(self): - w = initializers.random_matrix(([5] * 3, None)) - A = initializers.random_matrix(([5] * 3, [5] * 3)) - x = initializers.random_matrix(([5] * 3, None)) - z = initializers.random_matrix(([5] * 3, None)) + w = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) + A = initializers.random_matrix(([5] * 3, [5] * 3), dtype=self.dtype) + x = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) + z = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) def func1(x): return 0.5 * ops.flat_inner(x, w) ** 2 @@ -36,10 +36,10 @@ def func2(x): np.testing.assert_allclose(actual_v2, desired_v2, rtol=1e-4) def testHessianVectorProduct(self): - w = initializers.random_matrix(([5] * 3, None)) - A = initializers.random_matrix(([5] * 3, [5] * 3)) - x = initializers.random_matrix(([5] * 3, None)) - z = initializers.random_matrix(([5] * 3, None)) + w = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) + A = initializers.random_matrix(([5] * 3, [5] * 3), dtype=self.dtype) + x = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) + z = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) projected_vector = ops.full(riemannian.project(z, x)) def func1(x): @@ -68,6 +68,14 @@ def func2(x): np.testing.assert_allclose(actual2_v, desired2_v, rtol=1e-3) +class AutodiffTestFloat32(tf.test.TestCase, _AutodiffTest): + dtype = tf.float32 + + +class AutodiffTestFloat64(tf.test.TestCase, _AutodiffTest): + dtype = tf.float64 + + if __name__ == "__main__": tf.test.main() From e4c6fd4bdc5ec309a62f14f8b554929a93f1d6b3 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 22:06:15 +0000 Subject: [PATCH 101/233] Remove some code duplication --- t3f/autodiff_test.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index 2287dfa0..21de2acc 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -40,7 +40,7 @@ def testHessianVectorProduct(self): A = initializers.random_matrix(([5] * 3, [5] * 3), dtype=self.dtype) x = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) z = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) - projected_vector = ops.full(riemannian.project(z, x)) + projected_vector = riemannian.project(z, x) def func1(x): return 0.5 * ops.flat_inner(x, w) ** 2 @@ -49,8 +49,7 @@ def func1(x): # Hessian by vector: w actual1 = ops.full(autodiff.hessian_vector_product(func1, x, z)) - projected_z = riemannian.project(z, x) - desired1 = riemannian.project(ops.flat_inner(projected_z, w) * w, x) + desired1 = riemannian.project(ops.flat_inner(projected_vector, w) * w, x) desired1 = ops.full(desired1) with self.test_session() as sess: actual1_v, desired1_v = sess.run([actual1, desired1]) @@ -60,7 +59,6 @@ def func2(x): return ops.quadratic_form(A, x, x) # Hessian of is A + A.T actual2 = ops.full(autodiff.hessian_vector_product(func2, x, z)) - projected_vector = riemannian.project(z, x) hessian_by_vector = ops.matmul(ops.transpose(A) + A, projected_vector) desired2 = ops.full(riemannian.project(hessian_by_vector, x)) with self.test_session() as sess: From 9ff88319e25b2acd1668b08449455ae0c986726e Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 22:07:49 +0000 Subject: [PATCH 102/233] Remove @ notataion (make python 2 compatible) --- t3f/autodiff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 46316f6f..f9882f16 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -294,7 +294,7 @@ def _block_diag_hessian_vector_product(func, x, vector): else: curr_grad = cores_grad[r1:, :, :r2] if i < x.ndims() - 1: - proj = (tf.eye(r1 * n) - q @ tf.transpose(q)) + proj = tf.matmul(tf.eye(r1 * n) - q, tf.transpose(q)) # TODO: multiply faster. delta = tf.matmul(proj, tf.reshape(curr_grad, (-1, r2))) delta = tf.reshape(delta, left.tt_cores[i].shape) From 33c3d0017c79c7352c584451be9b9a0dc1a3e220 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 22:18:00 +0000 Subject: [PATCH 103/233] move converts to and from deltas into riemannain --- t3f/__init__.py | 3 +- t3f/autodiff.py | 132 +++------------------------------------------- t3f/riemannian.py | 118 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 127 deletions(-) diff --git a/t3f/__init__.py b/t3f/__init__.py index bc07724d..5cfe5027 100644 --- a/t3f/__init__.py +++ b/t3f/__init__.py @@ -50,6 +50,8 @@ from t3f.riemannian import project from t3f.riemannian import project_matmul from t3f.riemannian import project_sum +from t3f.riemannian import tangent_space_to_deltas +from t3f.riemannian import deltas_to_tangent_space from t3f.shapes import batch_size from t3f.shapes import clean_raw_shape @@ -69,7 +71,6 @@ from t3f.decompositions import to_tt_matrix from t3f.decompositions import to_tt_tensor -from t3f.autodiff import tangent_space_to_deltas from t3f.autodiff import gradients from t3f.autodiff import hessian_vector_product diff --git a/t3f/autodiff.py b/t3f/autodiff.py index f9882f16..5ca1873f 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -7,126 +7,6 @@ from t3f import riemannian -# TODO: move to riemannian. -def tangent_space_to_deltas(tt, name='t3f_tangent_space_to_deltas'): - """Convert an element of the tangent space to deltas representation. - - Tangent space elements (outputs of t3f.project) look like: - dP1 V2 ... Vd + U1 dP2 V3 ... Vd + ... + U1 ... Ud-1 dPd. - - This function takes as input an element of the tangent space and converts - it to the list of deltas [dP1, ..., dPd]. - - Args: - tt: `TensorTrain` or `TensorTrainBatch` that is a result of t3f.project, - t3f.project_matmul, or other similar functions. - name: string, name of the Op. - - Returns: - A list of delta-cores. - """ - if not hasattr(tt, 'projection_on') or tt.projection_on is None: - raise ValueError('tt argument is supposed to be a projection, but it ' - 'lacks projection_on field') - num_dims = tt.ndims() - left_tt_rank_dim = tt.left_tt_rank_dim - right_tt_rank_dim = tt.right_tt_rank_dim - deltas = [None] * num_dims - tt_ranks = shapes.lazy_tt_ranks(tt) - for i in range(1, num_dims - 1): - if int(tt_ranks[i] / 2) != tt_ranks[i] / 2: - raise ValueError('tt argument is supposed to be a projection, but its ' - 'ranks are not even.') - with tf.name_scope(name, values=tt.tt_cores): - for i in range(1, num_dims - 1): - r1, r2 = tt_ranks[i], tt_ranks[i + 1] - curr_core = tt.tt_cores[i] - slc = [slice(None)] * len(curr_core.shape) - slc[left_tt_rank_dim] = slice(int(r1 / 2), None) - slc[right_tt_rank_dim] = slice(0, int(r2 / 2)) - deltas[i] = curr_core[slice] - slc = [slice(None)] * len(tt.tt_cores[0].shape) - slc[right_tt_rank_dim] = slice(0, int(tt_ranks[1] / 2)) - deltas[0] = tt.tt_cores[0][slc] - slc = [slice(None)] * len(tt.tt_cores[0].shape) - slc[left_tt_rank_dim] = slice(int(tt_ranks[-1] / 2), None) - deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][slc] - return deltas - - -# TODO: move to riemannian. -def deltas_to_tangent_space(deltas, tt, left=None, right=None, - name='t3f_deltas_to_tangent_space'): - """Converts deltas representation of tangent space vector to TensorTrain object. - - Takes as input a list of [dP1, ..., dPd] and returns - dP1 V2 ... Vd + U1 dP2 V3 ... Vd + ... + U1 ... Ud-1 dPd. - - This function is hard to use correctly because deltas should abey the - so called gauge conditions. If the don't, the function will silently return - incorrect result. This is why this function is not imported in the __init__. - - Args: - deltas: a list of deltas (essentially TT-cores). - tt: `TensorTrain` object on which the tangent space tensor represented by - delta is projected. - left: t3f.orthogonilize_tt_cores(tt). If you have it already compute, you - may pass it as argument to avoid recomputing. - right: t3f.orthogonilize_tt_cores(left, left_to_right=False). If you have - it already compute, you may pass it as argument to avoid recomputing. - name: string, name of the Op. - - Returns: - `TensorTrain` object constructed from deltas, that is from the tangent - space at point `tt`. - """ - cores = [] - dtype = deltas[0].dtype - num_dims = left.ndims() - # TODO: add cache instead of mannually pasisng precomputed stuff? - if left is None: - left = decompositions.orthogonalize_tt_cores(tt) - if right is None: - right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) - left_tangent_tt_ranks = shapes.lazy_tt_ranks(left) - right_tangent_tt_ranks = shapes.lazy_tt_ranks(left) - raw_shape = shapes.lazy_raw_shape(left) - right_rank_dim = left.right_tt_rank_dim - left_rank_dim = left.left_tt_rank_dim - is_batch_case = hasattr(tt, 'batch_size') - for i in range(num_dims): - left_tt_core = left.tt_cores[i] - right_tt_core = right.tt_cores[i] - - if i == 0: - tangent_core = tf.concat((deltas[i], left_tt_core), - axis=right_rank_dim) - elif i == num_dims - 1: - tangent_core = tf.concat((right_tt_core, deltas[i]), - axis=left_rank_dim) - else: - rank_1 = right_tangent_tt_ranks[i] - rank_2 = left_tangent_tt_ranks[i + 1] - if tt.is_tt_matrix(): - mode_size_n = raw_shape[0][i] - mode_size_m = raw_shape[1][i] - shape = [rank_1, mode_size_n, mode_size_m, rank_2] - else: - mode_size_n = raw_shape[0][i] - shape = [rank_1, mode_size_n, rank_2] - if is_batch_case: - shape = [tt.batch_size] + shape - zeros = tf.zeros(shape, dtype=dtype) - upper = tf.concat((right_tt_core, zeros), axis=right_rank_dim) - lower = tf.concat((deltas[i], left_tt_core), axis=right_rank_dim) - tangent_core = tf.concat((upper, lower), axis=left_rank_dim) - cores.append(tangent_core) - # Create instance of the same class as tt. - tangent = tt.__class__(cores) - tangent.projection_on = tt - return tangent - - def _gradients(func, x, x_projection, left, right): """Internal version of t3f.gradients that assumes some precomputed inputs.""" h = func(x_projection) @@ -160,7 +40,7 @@ def _gradients(func, x, x_projection, left, right): else: delta = curr_grad deltas.append(delta) - return deltas_to_tangent_space(deltas, x, left, right) + return riemannian.deltas_to_tangent_space(deltas, x, left, right) def gradients(func, x, name='t3f_gradients'): @@ -199,7 +79,7 @@ def gradients(func, x, name='t3f_gradients'): left = decompositions.orthogonalize_tt_cores(x) right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]] - x_projection = deltas_to_tangent_space(deltas, x, left, right) + x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) return _gradients(func, x, x_projection, left, right) @@ -265,11 +145,11 @@ def _block_diag_hessian_vector_product(func, x, vector): left = decompositions.orthogonalize_tt_cores(x) right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]] - x_projection = deltas_to_tangent_space(deltas, x, left, right) + x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) grad = _gradients(func, x, x_projection, left, right) vector_projected = riemannian.project(vector, x) - vector_deltas = tangent_space_to_deltas(vector_projected) - grad_deltas = tangent_space_to_deltas(grad) + vector_deltas = riemannian.tangent_space_to_deltas(vector_projected) + grad_deltas = riemannian.tangent_space_to_deltas(grad) final_deltas = [] for i in range(x.ndims()): h = tf.reduce_sum(grad_deltas[i] * vector_deltas[i]) @@ -301,4 +181,4 @@ def _block_diag_hessian_vector_product(func, x, vector): else: delta = curr_grad final_deltas.append(delta) - return deltas_to_tangent_space(final_deltas, x, left, right) + return riemannian.deltas_to_tangent_space(final_deltas, x, left, right) diff --git a/t3f/riemannian.py b/t3f/riemannian.py index a3faf434..e1e65151 100644 --- a/t3f/riemannian.py +++ b/t3f/riemannian.py @@ -749,3 +749,121 @@ def slice_tt_core(tt_core, left_idx, right_idx): # Maintain the projection_on property. res.projection_on = tt_objects[0].projection_on return res + + +def tangent_space_to_deltas(tt, name='t3f_tangent_space_to_deltas'): + """Convert an element of the tangent space to deltas representation. + + Tangent space elements (outputs of t3f.project) look like: + dP1 V2 ... Vd + U1 dP2 V3 ... Vd + ... + U1 ... Ud-1 dPd. + + This function takes as input an element of the tangent space and converts + it to the list of deltas [dP1, ..., dPd]. + + Args: + tt: `TensorTrain` or `TensorTrainBatch` that is a result of t3f.project, + t3f.project_matmul, or other similar functions. + name: string, name of the Op. + + Returns: + A list of delta-cores. + """ + if not hasattr(tt, 'projection_on') or tt.projection_on is None: + raise ValueError('tt argument is supposed to be a projection, but it ' + 'lacks projection_on field') + num_dims = tt.ndims() + left_tt_rank_dim = tt.left_tt_rank_dim + right_tt_rank_dim = tt.right_tt_rank_dim + deltas = [None] * num_dims + tt_ranks = shapes.lazy_tt_ranks(tt) + for i in range(1, num_dims - 1): + if int(tt_ranks[i] / 2) != tt_ranks[i] / 2: + raise ValueError('tt argument is supposed to be a projection, but its ' + 'ranks are not even.') + with tf.name_scope(name, values=tt.tt_cores): + for i in range(1, num_dims - 1): + r1, r2 = tt_ranks[i], tt_ranks[i + 1] + curr_core = tt.tt_cores[i] + slc = [slice(None)] * len(curr_core.shape) + slc[left_tt_rank_dim] = slice(int(r1 / 2), None) + slc[right_tt_rank_dim] = slice(0, int(r2 / 2)) + deltas[i] = curr_core[slice] + slc = [slice(None)] * len(tt.tt_cores[0].shape) + slc[right_tt_rank_dim] = slice(0, int(tt_ranks[1] / 2)) + deltas[0] = tt.tt_cores[0][slc] + slc = [slice(None)] * len(tt.tt_cores[0].shape) + slc[left_tt_rank_dim] = slice(int(tt_ranks[-1] / 2), None) + deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][slc] + return deltas + + +def deltas_to_tangent_space(deltas, tt, left=None, right=None, + name='t3f_deltas_to_tangent_space'): + """Converts deltas representation of tangent space vector to TensorTrain object. + + Takes as input a list of [dP1, ..., dPd] and returns + dP1 V2 ... Vd + U1 dP2 V3 ... Vd + ... + U1 ... Ud-1 dPd. + + This function is hard to use correctly because deltas should abey the + so called gauge conditions. If the don't, the function will silently return + incorrect result. This is why this function is not imported in the __init__. + + Args: + deltas: a list of deltas (essentially TT-cores). + tt: `TensorTrain` object on which the tangent space tensor represented by + delta is projected. + left: t3f.orthogonilize_tt_cores(tt). If you have it already compute, you + may pass it as argument to avoid recomputing. + right: t3f.orthogonilize_tt_cores(left, left_to_right=False). If you have + it already compute, you may pass it as argument to avoid recomputing. + name: string, name of the Op. + + Returns: + `TensorTrain` object constructed from deltas, that is from the tangent + space at point `tt`. + """ + cores = [] + dtype = deltas[0].dtype + num_dims = left.ndims() + # TODO: add cache instead of mannually pasisng precomputed stuff? + if left is None: + left = decompositions.orthogonalize_tt_cores(tt) + if right is None: + right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) + left_tangent_tt_ranks = shapes.lazy_tt_ranks(left) + right_tangent_tt_ranks = shapes.lazy_tt_ranks(left) + raw_shape = shapes.lazy_raw_shape(left) + right_rank_dim = left.right_tt_rank_dim + left_rank_dim = left.left_tt_rank_dim + is_batch_case = hasattr(tt, 'batch_size') + for i in range(num_dims): + left_tt_core = left.tt_cores[i] + right_tt_core = right.tt_cores[i] + + if i == 0: + tangent_core = tf.concat((deltas[i], left_tt_core), + axis=right_rank_dim) + elif i == num_dims - 1: + tangent_core = tf.concat((right_tt_core, deltas[i]), + axis=left_rank_dim) + else: + rank_1 = right_tangent_tt_ranks[i] + rank_2 = left_tangent_tt_ranks[i + 1] + if tt.is_tt_matrix(): + mode_size_n = raw_shape[0][i] + mode_size_m = raw_shape[1][i] + shape = [rank_1, mode_size_n, mode_size_m, rank_2] + else: + mode_size_n = raw_shape[0][i] + shape = [rank_1, mode_size_n, rank_2] + if is_batch_case: + shape = [tt.batch_size] + shape + zeros = tf.zeros(shape, dtype=dtype) + upper = tf.concat((right_tt_core, zeros), axis=right_rank_dim) + lower = tf.concat((deltas[i], left_tt_core), axis=right_rank_dim) + tangent_core = tf.concat((upper, lower), axis=left_rank_dim) + cores.append(tangent_core) + # Create instance of the same class as tt. + tangent = tt.__class__(cores) + tangent.projection_on = tt + return tangent From fb7c54bad5c7dfe275d30b8b462f3a5db2cd7512 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 30 Oct 2018 22:23:11 +0000 Subject: [PATCH 104/233] make it compatible with older tf versions --- t3f/autodiff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 5ca1873f..2686807d 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -10,7 +10,7 @@ def _gradients(func, x, x_projection, left, right): """Internal version of t3f.gradients that assumes some precomputed inputs.""" h = func(x_projection) - cores_grad = tf.gradients(h, x_projection.tt_cores) + cores_grad = tf.gradients(h, list(x_projection.tt_cores)) deltas = [] for i in range(x.ndims()): if x.is_tt_matrix(): From 30c0c6dba4ec129c2141e083d8155b6c81d79a71 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 31 Oct 2018 20:02:18 +0000 Subject: [PATCH 105/233] Simplify hessian by vec logic --- t3f/autodiff.py | 78 ++++++++++++++++++++----------------------------- 1 file changed, 31 insertions(+), 47 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 2686807d..ec2c0848 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -7,47 +7,31 @@ from t3f import riemannian -def _gradients(func, x, x_projection, left, right): - """Internal version of t3f.gradients that assumes some precomputed inputs.""" - h = func(x_projection) - cores_grad = tf.gradients(h, list(x_projection.tt_cores)) - deltas = [] - for i in range(x.ndims()): - if x.is_tt_matrix(): +def _enforce_gauge_conditions(deltas, left): + """Project deltas that define tangent space vec onto the gauge conditions.""" + proj_deltas = [] + for i in range(left.ndims()): + if left.is_tt_matrix(): r1, n, m, r2 = left.tt_cores[i].shape.as_list() else: r1, n, r2 = left.tt_cores[i].shape.as_list() q = tf.reshape(left.tt_cores[i], (-1, r2)) - if x.is_tt_matrix(): - if i == 0: - curr_grad = cores_grad[i][:, :, :, :r2] - elif i == x.ndims() - 1: - curr_grad = cores_grad[i][r1:, :, :, :] - else: - curr_grad = cores_grad[i][r1:, :, :, :r2] - else: - if i == 0: - curr_grad = cores_grad[i][:, :, :r2] - elif i == w.ndims() - 1: - curr_grad = cores_grad[i][r1:, :, :] - else: - curr_grad = cores_grad[i][r1:, :, :r2] - if i < x.ndims() - 1: - delta = curr_grad - delta = tf.reshape(delta, (-1, r2)) - delta -= tf.matmul(q, tf.matmul(tf.transpose(q), delta)) - delta = tf.reshape(delta, left.tt_cores[i].shape) + if i < left.ndims() - 1: + proj_delta = deltas[i] + proj_delta = tf.reshape(proj_delta, (-1, r2)) + proj_delta -= tf.matmul(q, tf.matmul(tf.transpose(q), proj_delta)) + proj_delta = tf.reshape(proj_delta, left.tt_cores[i].shape) else: - delta = curr_grad - deltas.append(delta) - return riemannian.deltas_to_tangent_space(deltas, x, left, right) + proj_delta = deltas[i] + proj_deltas.append(proj_delta) + return proj_deltas -def gradients(func, x, name='t3f_gradients'): +def gradients(func, x, name='t3f_gradients', debug=True): """Riemannian autodiff: returns gradient projected on tangent space of TT. - Automatically computes projection of the gradient df/dx onto the - tangent space of TT tensor at point x. + Computes projection of the gradient df/dx onto the tangent space of TT tensor + at point x. Warning: this is experimental feature and it may not work for some function, e.g. ones that include QR or SVD decomposition (t3f.project, t3f.round) or @@ -80,19 +64,21 @@ def gradients(func, x, name='t3f_gradients'): right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]] x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) - return _gradients(func, x, x_projection, left, right) + cores_grad = tf.gradients(func(x_projection), deltas) + deltas = _enforce_gauge_conditions(cores_grad, left) + return riemannian.deltas_to_tangent_space(deltas, x, left, right) def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product'): """P_x d^2f/dx^2 P_x vector, i.e. Riemannian hessian by vector product. - Automatically computes + Computes P_x d^2f/dx^2 P_x vector where P_x is projection onto the tangent space of TT at point x and d^2f/dx^2 is the Hessian of the function. - Note that the true hessian also includes the manifold curvature term - which is ignored here. + Note that the true Riemannian hessian also includes the manifold curvature + term which is ignored here. Warning: this is experimental feature and it may not work for some function, e.g. ones that include QR or SVD decomposition (t3f.project, t3f.round) or @@ -127,17 +113,15 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product'): left = decompositions.orthogonalize_tt_cores(x) right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) vector_projected = riemannian.project(vector, x) - vector_projected = shapes.expand_batch_dim(vector_projected) - vector_projected.projection_on = x - - def new_f(new_x): - grad = _gradients(func, x, new_x, left, right) - grad = shapes.expand_batch_dim(grad) - # TODO: durty hack. - grad.projection_on = x - return riemannian.pairwise_flat_inner_projected(grad, vector_projected)[0, 0] - - return gradients(new_f, x) + deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]] + x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) + cores_grad = tf.gradients(func(x_projection), deltas) + vec_deltas = riemannian.tangent_space_to_deltas(vector_projected) + products = [tf.reduce_sum(a * b) for a, b in zip(cores_grad, vec_deltas)] + grad_times_vec = tf.add_n(products) + second_cores_grad = tf.gradients(grad_times_vec, deltas) + final_deltas = _enforce_gauge_conditions(second_cores_grad, left) + return riemannian.deltas_to_tangent_space(final_deltas, x, left, right) def _block_diag_hessian_vector_product(func, x, vector): From 2f4f433839622ff89c81304552ded4397f46a782 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 31 Oct 2018 20:02:53 +0000 Subject: [PATCH 106/233] Fix tangent_space_to_deltas bug fix with indexing --- t3f/riemannian.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/riemannian.py b/t3f/riemannian.py index e1e65151..445e8086 100644 --- a/t3f/riemannian.py +++ b/t3f/riemannian.py @@ -787,12 +787,12 @@ def tangent_space_to_deltas(tt, name='t3f_tangent_space_to_deltas'): slc = [slice(None)] * len(curr_core.shape) slc[left_tt_rank_dim] = slice(int(r1 / 2), None) slc[right_tt_rank_dim] = slice(0, int(r2 / 2)) - deltas[i] = curr_core[slice] + deltas[i] = curr_core[slc] slc = [slice(None)] * len(tt.tt_cores[0].shape) slc[right_tt_rank_dim] = slice(0, int(tt_ranks[1] / 2)) deltas[0] = tt.tt_cores[0][slc] slc = [slice(None)] * len(tt.tt_cores[0].shape) - slc[left_tt_rank_dim] = slice(int(tt_ranks[-1] / 2), None) + slc[left_tt_rank_dim] = slice(int(tt_ranks[-2] / 2), None) deltas[num_dims - 1] = tt.tt_cores[num_dims - 1][slc] return deltas From 638ced8fa6bf093c6effb0f2eb49b9ea5b2172fd Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 31 Oct 2018 20:19:42 +0000 Subject: [PATCH 107/233] Test that fails on functions that are not proper --- t3f/autodiff_test.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index 21de2acc..6903998b 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -35,6 +35,16 @@ def func2(x): actual_v2, desired_v2 = sess.run([actual2, desired2]) np.testing.assert_allclose(actual_v2, desired_v2, rtol=1e-4) + def func3(x): + # A function which is not invariant to different representations of the + # same tensor, i.e. it does not even have a Riemannian gradient. + return tf.add_n([tf.reduce_sum(c) for c in x.tt_cores]) ** 2 + actual3 = ops.full(autodiff.gradients(func3, x)) + with self.assertRaises(tf.errors.InvalidArgumentError): + with self.test_session() as sess: + sess.run(actual3) + + def testHessianVectorProduct(self): w = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) A = initializers.random_matrix(([5] * 3, [5] * 3), dtype=self.dtype) @@ -65,6 +75,16 @@ def func2(x): actual2_v, desired2_v = sess.run([actual2, desired2]) np.testing.assert_allclose(actual2_v, desired2_v, rtol=1e-3) + def func3(x): + # A function which is not invariant to different representations of the + # same tensor, i.e. it does not even have a Riemannian gradient or + # hessian. + return tf.add_n([tf.reduce_sum(c) for c in x.tt_cores]) ** 2 + actual3 = ops.full(autodiff.hessian_vector_product(func3, x, z)) + with self.assertRaises(tf.errors.InvalidArgumentError): + with self.test_session() as sess: + sess.run(actual3) + class AutodiffTestFloat32(tf.test.TestCase, _AutodiffTest): dtype = tf.float32 From fb43d1f4a64b77f8b64134460b0c77cb5f79e185 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 31 Oct 2018 20:32:03 +0000 Subject: [PATCH 108/233] Init function check implementation --- t3f/autodiff.py | 67 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index ec2c0848..984d6092 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -27,6 +27,36 @@ def _enforce_gauge_conditions(deltas, left): return proj_deltas +def _is_invariant_to_input_transforms(f_value_1, f_value_2, + name="check_autodiff_arguments"): + """Returns an assert op that checks that the f_value_1 == f_value_2. + + Args: + f_value_1: tf.Tensor, value of the function computed on x_1 + f_value_2: tf.Tensor, value of the function computed on x_2 + name: String, the name of the returned op + + Here we assume that as tensors x_1 == x_2, but their TT-cores are different, + e.g. x_2 is a cores orthogonalization version of x_1. + + The function prints a warning about introducing overhead and returns an Assert + op that checks that the two values are reasonably close to each other. + + Returns: + tf.op, assertion operation. + """ + print('Warning: debug mode of Riemannian autodiff is turned on which ' + 'makes things a bit slower. It is advisable to keep debug=True untill ' + 'actuall production usage, since debug mode does help to catch bugs.') + rel_diff = tf.abs((f_value_1 - f_value_2) / f_value_1) + err_msg = "The function passed to Riemannian autodiff returns different " \ + "values for two different versions of the same tensor. " \ + "The function values are" + assert_op = tf.Assert(rel_diff < 1e-5, [err_msg, f_value_1, f_value_2], + name=name) + return assert_op + + def gradients(func, x, name='t3f_gradients', debug=True): """Riemannian autodiff: returns gradient projected on tangent space of TT. @@ -51,6 +81,10 @@ def gradients(func, x, name='t3f_gradients', debug=True): x: point at which to compute the gradient and on which tangent space to project the gradient. name: string, name of the Op. + debug: [True] whether to do a sanity check that the passed function is + invariant to different TT representations (otherwise the Rieamnnian + gradient doesn't even exist). It makes things slower, but helps catching + bugs, so turn it off during production deployment. Returns: `TensorTrain`, projection of the gradient df/dx onto the tangent space at @@ -62,14 +96,22 @@ def gradients(func, x, name='t3f_gradients', debug=True): with tf.name_scope(name, values=x.tt_cores): left = decompositions.orthogonalize_tt_cores(x) right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) - deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]] + deltas = [right.tt_cores[0]] + deltas += [tf.zeros_like(cc) for cc in right.tt_cores[1:]] x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) - cores_grad = tf.gradients(func(x_projection), deltas) + function_value = func(x_projection) + if debug: + assert_op = _is_invariant_to_input_transforms(function_value, func(right)) + else: + assert_op = None + with tf.control_dependencies([assert_op]): + cores_grad = tf.gradients(function_value, deltas) deltas = _enforce_gauge_conditions(cores_grad, left) return riemannian.deltas_to_tangent_space(deltas, x, left, right) -def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product'): +def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', + debug=True): """P_x d^2f/dx^2 P_x vector, i.e. Riemannian hessian by vector product. Computes @@ -99,7 +141,11 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product'): x: point at which to compute the Hessian and on which tangent space to project the gradient. vector: `TensorTrain` object which to multiply be the Hessian. - name: string, name of the Op. + name: string, name of the Op. + debug: [True] whether to do a sanity check that the passed function is + invariant to different TT representations (otherwise the Rieamnnian + gradient doesn't even exist). It makes things slower, but helps catching + bugs, so turn it off during production deployment. Returns: `TensorTrain`, projection of the gradient df/dx onto the tangent space at @@ -112,10 +158,17 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product'): with tf.name_scope(name, values=all_cores): left = decompositions.orthogonalize_tt_cores(x) right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) - vector_projected = riemannian.project(vector, x) - deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]] + deltas = [right.tt_cores[0]] + deltas += [tf.zeros_like(cc) for cc in right.tt_cores[1:]] x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) - cores_grad = tf.gradients(func(x_projection), deltas) + function_value = func(x_projection) + if debug: + assert_op = _is_invariant_to_input_transforms(function_value, func(right)) + else: + assert_op = None + with tf.control_dependencies([assert_op]): + vector_projected = riemannian.project(vector, x) + cores_grad = tf.gradients(function_value, deltas) vec_deltas = riemannian.tangent_space_to_deltas(vector_projected) products = [tf.reduce_sum(a * b) for a, b in zip(cores_grad, vec_deltas)] grad_times_vec = tf.add_n(products) From 230e7f2ecfcaa839b7dcf6c24077b4274207c2fc Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 3 Nov 2018 18:44:37 +0000 Subject: [PATCH 109/233] Try if other version of TF do not fail --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index e075c33c..6656268d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,10 @@ python: env: matrix: - TF_VERSION=1.0 + - TF_VERSION=1.1 + - TF_VERSION=1.2 + - TF_VERSION=1.3 + - TF_VERSION=1.4 - TF_VERSION=1.10 - TF_VERSION=1.11 # command to install dependencies From 00f66b3ce24cf1d36d45d77c41a1fae3fe9b7386 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 3 Nov 2018 19:37:34 +0000 Subject: [PATCH 110/233] Requere tf >= 1.2 so that the tests pass --- .travis.yml | 4 ---- README.md | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6656268d..08ad9e23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,11 +7,7 @@ python: - "3.6" env: matrix: - - TF_VERSION=1.0 - - TF_VERSION=1.1 - TF_VERSION=1.2 - - TF_VERSION=1.3 - - TF_VERSION=1.4 - TF_VERSION=1.10 - TF_VERSION=1.11 # command to install dependencies diff --git a/README.md b/README.md index 16a89841..2bec3357 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ TensorFlow implementation of the Tensor Train (TT) -Toolbox. API is available via [readthedocs](https://t3f.readthedocs.io/en/latest/). # Installation -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, tested versions are from 1.0 to 1.11 (see [here](https://www.tensorflow.org/install/) for TF installation instructions). +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation of version >= 1.2, tested versions are from 1.2 to 1.11 (see [here](https://www.tensorflow.org/install/) for TF installation instructions). We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. Then simply run ```bash From 63ef05d691f648a51a0ff022b6a94de7752aa726 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 4 Nov 2018 14:56:40 +0000 Subject: [PATCH 111/233] Deltas to/from tangent space tests --- t3f/riemannian_test.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/t3f/riemannian_test.py b/t3f/riemannian_test.py index df3f5246..3055d831 100644 --- a/t3f/riemannian_test.py +++ b/t3f/riemannian_test.py @@ -274,6 +274,50 @@ def testWeightedAddNProjectedBatch(self): desired_val, actual_val = sess.run((desired, actual)) self.assertAllClose(desired_val, actual_val, atol=1e-5, rtol=1e-5) + def testToAndFromDeltas(self): + # Test converting to and from deltas representation of the tangent space + # element. + what = initializers.random_tensor((2, 3, 4), 4, dtype=self.dtype) + where = initializers.random_tensor((2, 3, 4), 3, dtype=self.dtype) + projected = riemannian.project(what, where) + + deltas = riemannian.tangent_space_to_deltas(projected) + reconstructed_projected = riemannian.deltas_to_tangent_space(deltas, where) + # Tangent space element norm can be computed from deltas norm. + projected_normsq_desired = ops.frobenius_norm_squared(projected) + projected_normsq_actual = tf.add_n([tf.reduce_sum(c * c) for c in deltas]) + with self.test_session() as sess: + desired_val, actual_val = sess.run((ops.full(projected), + ops.full(reconstructed_projected))) + self.assertAllClose(desired_val, actual_val) + desired_val, actual_val = sess.run((projected_normsq_desired, + projected_normsq_actual)) + self.assertAllClose(desired_val, actual_val) + + def testToAndFromDeltasBatch(self): + # Test converting to and from deltas representation of the tangent space + # element in the batch case. + what = initializers.random_matrix_batch(((2, 3, 4), (3, 3, 3)), 4, + batch_size=3, dtype=self.dtype) + where = initializers.random_matrix(((2, 3, 4), (3, 3, 3)), 3, + dtype=self.dtype) + projected = riemannian.project(what, where) + + deltas = riemannian.tangent_space_to_deltas(projected) + reconstructed_projected = riemannian.deltas_to_tangent_space(deltas, where) + # Tangent space element norm can be computed from deltas norm. + projected_normsq_desired = ops.frobenius_norm_squared(projected) + d_normssq = [tf.reduce_sum(tf.reshape(c, (3, -1)) ** 2, 1) for c in deltas] + projected_normsq_actual = tf.add_n(d_normssq) + + with self.test_session() as sess: + desired_val, actual_val = sess.run((ops.full(projected), + ops.full(reconstructed_projected))) + self.assertAllClose(desired_val, actual_val) + desired_val, actual_val = sess.run((projected_normsq_desired, + projected_normsq_actual)) + self.assertAllClose(desired_val, actual_val) + class RiemannianTestFloat32(tf.test.TestCase, _RiemannianTest): dtype = tf.float32 From 2d90b1f11b02e3d3cca032a79f22e009a012b878 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 4 Nov 2018 14:57:35 +0000 Subject: [PATCH 112/233] Fix batch deltas to/from tangent space --- t3f/riemannian.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/t3f/riemannian.py b/t3f/riemannian.py index 445e8086..fe244db2 100644 --- a/t3f/riemannian.py +++ b/t3f/riemannian.py @@ -766,7 +766,7 @@ def tangent_space_to_deltas(tt, name='t3f_tangent_space_to_deltas'): name: string, name of the Op. Returns: - A list of delta-cores. + A list of delta-cores (tf.Tensors). """ if not hasattr(tt, 'projection_on') or tt.projection_on is None: raise ValueError('tt argument is supposed to be a projection, but it ' @@ -799,17 +799,18 @@ def tangent_space_to_deltas(tt, name='t3f_tangent_space_to_deltas'): def deltas_to_tangent_space(deltas, tt, left=None, right=None, name='t3f_deltas_to_tangent_space'): - """Converts deltas representation of tangent space vector to TensorTrain object. + """Converts deltas representation of tangent space vector to TT object. Takes as input a list of [dP1, ..., dPd] and returns dP1 V2 ... Vd + U1 dP2 V3 ... Vd + ... + U1 ... Ud-1 dPd. This function is hard to use correctly because deltas should abey the so called gauge conditions. If the don't, the function will silently return - incorrect result. This is why this function is not imported in the __init__. + incorrect result. This is why this function is not imported in __init__. Args: - deltas: a list of deltas (essentially TT-cores). + deltas: a list of deltas (essentially TT-cores) obeying the gauge + conditions. tt: `TensorTrain` object on which the tangent space tensor represented by delta is projected. left: t3f.orthogonilize_tt_cores(tt). If you have it already compute, you @@ -823,8 +824,8 @@ def deltas_to_tangent_space(deltas, tt, left=None, right=None, space at point `tt`. """ cores = [] - dtype = deltas[0].dtype - num_dims = left.ndims() + dtype = tt.dtype + num_dims = tt.ndims() # TODO: add cache instead of mannually pasisng precomputed stuff? if left is None: left = decompositions.orthogonalize_tt_cores(tt) @@ -835,10 +836,19 @@ def deltas_to_tangent_space(deltas, tt, left=None, right=None, raw_shape = shapes.lazy_raw_shape(left) right_rank_dim = left.right_tt_rank_dim left_rank_dim = left.left_tt_rank_dim - is_batch_case = hasattr(tt, 'batch_size') + is_batch_case = len(deltas[0].shape) > len(tt.tt_cores[0].shape) + if is_batch_case: + right_rank_dim += 1 + left_rank_dim += 1 + batch_size = deltas[0].shape[0] for i in range(num_dims): left_tt_core = left.tt_cores[i] right_tt_core = right.tt_cores[i] + if is_batch_case: + tile = [1] * len(left_tt_core.shape) + tile = [batch_size] + tile + left_tt_core = tf.tile(left_tt_core[None, ...], tile) + right_tt_core = tf.tile(right_tt_core[None, ...], tile) if i == 0: tangent_core = tf.concat((deltas[i], left_tt_core), @@ -857,13 +867,15 @@ def deltas_to_tangent_space(deltas, tt, left=None, right=None, mode_size_n = raw_shape[0][i] shape = [rank_1, mode_size_n, rank_2] if is_batch_case: - shape = [tt.batch_size] + shape + shape = [batch_size] + shape zeros = tf.zeros(shape, dtype=dtype) upper = tf.concat((right_tt_core, zeros), axis=right_rank_dim) lower = tf.concat((deltas[i], left_tt_core), axis=right_rank_dim) tangent_core = tf.concat((upper, lower), axis=left_rank_dim) cores.append(tangent_core) - # Create instance of the same class as tt. - tangent = tt.__class__(cores) + if is_batch_case: + tangent = TensorTrainBatch(cores, batch_size=batch_size) + else: + tangent = TensorTrain(cores) tangent.projection_on = tt return tangent From fc377b88d918ce4c4a65377e5cd89ef2d4871c39 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 4 Nov 2018 14:58:22 +0000 Subject: [PATCH 113/233] Remove deltas to tangent space from init (it is too hard to use to be external, I guess) --- t3f/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/t3f/__init__.py b/t3f/__init__.py index 5cfe5027..0b497678 100644 --- a/t3f/__init__.py +++ b/t3f/__init__.py @@ -51,7 +51,6 @@ from t3f.riemannian import project_matmul from t3f.riemannian import project_sum from t3f.riemannian import tangent_space_to_deltas -from t3f.riemannian import deltas_to_tangent_space from t3f.shapes import batch_size from t3f.shapes import clean_raw_shape From 6cd250891b81d17e0da4191f68037aa72e5246a1 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 4 Nov 2018 15:07:44 +0000 Subject: [PATCH 114/233] reshape(a, (Dimension(1), 1)) doesn't work in older tf, fix --- t3f/riemannian.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/riemannian.py b/t3f/riemannian.py index fe244db2..14b8c85b 100644 --- a/t3f/riemannian.py +++ b/t3f/riemannian.py @@ -840,7 +840,7 @@ def deltas_to_tangent_space(deltas, tt, left=None, right=None, if is_batch_case: right_rank_dim += 1 left_rank_dim += 1 - batch_size = deltas[0].shape[0] + batch_size = deltas[0].shape.as_list()[0] for i in range(num_dims): left_tt_core = left.tt_cores[i] right_tt_core = right.tt_cores[i] From 893d074394255749c1450971903bac9a1351e714 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 4 Nov 2018 15:21:46 +0000 Subject: [PATCH 115/233] Move draft of _block_diag_hess_by_vec into a separate branch --- t3f/autodiff.py | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 984d6092..7ee4ec30 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -175,47 +175,3 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', second_cores_grad = tf.gradients(grad_times_vec, deltas) final_deltas = _enforce_gauge_conditions(second_cores_grad, left) return riemannian.deltas_to_tangent_space(final_deltas, x, left, right) - - -def _block_diag_hessian_vector_product(func, x, vector): - # TODO: - left = decompositions.orthogonalize_tt_cores(x) - right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) - deltas = [right.tt_cores[0]] + [tf.zeros_like(cc) for cc in right.tt_cores[1:]] - x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) - grad = _gradients(func, x, x_projection, left, right) - vector_projected = riemannian.project(vector, x) - vector_deltas = riemannian.tangent_space_to_deltas(vector_projected) - grad_deltas = riemannian.tangent_space_to_deltas(grad) - final_deltas = [] - for i in range(x.ndims()): - h = tf.reduce_sum(grad_deltas[i] * vector_deltas[i]) - cores_grad = tf.gradients(h, x_projection.tt_cores[i])[0] - if x.is_tt_matrix(): - r1, n, m, r2 = left.tt_cores[i].shape.as_list() - else: - r1, n, r2 = left.tt_cores[i].shape.as_list() - q = tf.reshape(left.tt_cores[i], (-1, r2)) - if x.is_tt_matrix(): - if i == 0: - curr_grad = cores_grad[:, :, :, :r2] - elif i == w.ndims() - 1: - curr_grad = cores_grad[r1:, :, :, :] - else: - curr_grad = cores_grad[r1:, :, :, :r2] - else: - if i == 0: - curr_grad = cores_grad[:, :, :r2] - elif i == x.ndims() - 1: - curr_grad = cores_grad[r1:, :, :] - else: - curr_grad = cores_grad[r1:, :, :r2] - if i < x.ndims() - 1: - proj = tf.matmul(tf.eye(r1 * n) - q, tf.transpose(q)) - # TODO: multiply faster. - delta = tf.matmul(proj, tf.reshape(curr_grad, (-1, r2))) - delta = tf.reshape(delta, left.tt_cores[i].shape) - else: - delta = curr_grad - final_deltas.append(delta) - return riemannian.deltas_to_tangent_space(final_deltas, x, left, right) From 5c604c1fd97948fd28cfd5e34a8ccd73125595d1 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 20:56:55 +0000 Subject: [PATCH 116/233] linter: name_scope, unused imports, indents --- t3f/autodiff.py | 7 +-- t3f/riemannian.py | 107 +++++++++++++++++++++++----------------------- 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 7ee4ec30..d4e69899 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -1,8 +1,5 @@ import tensorflow as tf -from t3f.tensor_train import TensorTrain -from t3f import shapes -from t3f import batch_ops from t3f import decompositions from t3f import riemannian @@ -35,10 +32,10 @@ def _is_invariant_to_input_transforms(f_value_1, f_value_2, f_value_1: tf.Tensor, value of the function computed on x_1 f_value_2: tf.Tensor, value of the function computed on x_2 name: String, the name of the returned op - + Here we assume that as tensors x_1 == x_2, but their TT-cores are different, e.g. x_2 is a cores orthogonalization version of x_1. - + The function prints a warning about introducing overhead and returns an Assert op that checks that the two values are reasonably close to each other. diff --git a/t3f/riemannian.py b/t3f/riemannian.py index 14b8c85b..8322dcdb 100644 --- a/t3f/riemannian.py +++ b/t3f/riemannian.py @@ -777,9 +777,9 @@ def tangent_space_to_deltas(tt, name='t3f_tangent_space_to_deltas'): deltas = [None] * num_dims tt_ranks = shapes.lazy_tt_ranks(tt) for i in range(1, num_dims - 1): - if int(tt_ranks[i] / 2) != tt_ranks[i] / 2: - raise ValueError('tt argument is supposed to be a projection, but its ' - 'ranks are not even.') + if int(tt_ranks[i] / 2) != tt_ranks[i] / 2: + raise ValueError('tt argument is supposed to be a projection, but its ' + 'ranks are not even.') with tf.name_scope(name, values=tt.tt_cores): for i in range(1, num_dims - 1): r1, r2 = tt_ranks[i], tt_ranks[i + 1] @@ -827,55 +827,56 @@ def deltas_to_tangent_space(deltas, tt, left=None, right=None, dtype = tt.dtype num_dims = tt.ndims() # TODO: add cache instead of mannually pasisng precomputed stuff? - if left is None: - left = decompositions.orthogonalize_tt_cores(tt) - if right is None: - right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) - left_tangent_tt_ranks = shapes.lazy_tt_ranks(left) - right_tangent_tt_ranks = shapes.lazy_tt_ranks(left) - raw_shape = shapes.lazy_raw_shape(left) - right_rank_dim = left.right_tt_rank_dim - left_rank_dim = left.left_tt_rank_dim - is_batch_case = len(deltas[0].shape) > len(tt.tt_cores[0].shape) - if is_batch_case: - right_rank_dim += 1 - left_rank_dim += 1 - batch_size = deltas[0].shape.as_list()[0] - for i in range(num_dims): - left_tt_core = left.tt_cores[i] - right_tt_core = right.tt_cores[i] + with tf.name_scope(name, values=tt.tt_cores): + if left is None: + left = decompositions.orthogonalize_tt_cores(tt) + if right is None: + right = decompositions.orthogonalize_tt_cores(left, left_to_right=False) + left_tangent_tt_ranks = shapes.lazy_tt_ranks(left) + right_tangent_tt_ranks = shapes.lazy_tt_ranks(left) + raw_shape = shapes.lazy_raw_shape(left) + right_rank_dim = left.right_tt_rank_dim + left_rank_dim = left.left_tt_rank_dim + is_batch_case = len(deltas[0].shape) > len(tt.tt_cores[0].shape) if is_batch_case: - tile = [1] * len(left_tt_core.shape) - tile = [batch_size] + tile - left_tt_core = tf.tile(left_tt_core[None, ...], tile) - right_tt_core = tf.tile(right_tt_core[None, ...], tile) - - if i == 0: - tangent_core = tf.concat((deltas[i], left_tt_core), - axis=right_rank_dim) - elif i == num_dims - 1: - tangent_core = tf.concat((right_tt_core, deltas[i]), - axis=left_rank_dim) - else: - rank_1 = right_tangent_tt_ranks[i] - rank_2 = left_tangent_tt_ranks[i + 1] - if tt.is_tt_matrix(): - mode_size_n = raw_shape[0][i] - mode_size_m = raw_shape[1][i] - shape = [rank_1, mode_size_n, mode_size_m, rank_2] - else: - mode_size_n = raw_shape[0][i] - shape = [rank_1, mode_size_n, rank_2] + right_rank_dim += 1 + left_rank_dim += 1 + batch_size = deltas[0].shape.as_list()[0] + for i in range(num_dims): + left_tt_core = left.tt_cores[i] + right_tt_core = right.tt_cores[i] if is_batch_case: - shape = [batch_size] + shape - zeros = tf.zeros(shape, dtype=dtype) - upper = tf.concat((right_tt_core, zeros), axis=right_rank_dim) - lower = tf.concat((deltas[i], left_tt_core), axis=right_rank_dim) - tangent_core = tf.concat((upper, lower), axis=left_rank_dim) - cores.append(tangent_core) - if is_batch_case: - tangent = TensorTrainBatch(cores, batch_size=batch_size) - else: - tangent = TensorTrain(cores) - tangent.projection_on = tt - return tangent + tile = [1] * len(left_tt_core.shape) + tile = [batch_size] + tile + left_tt_core = tf.tile(left_tt_core[None, ...], tile) + right_tt_core = tf.tile(right_tt_core[None, ...], tile) + + if i == 0: + tangent_core = tf.concat((deltas[i], left_tt_core), + axis=right_rank_dim) + elif i == num_dims - 1: + tangent_core = tf.concat((right_tt_core, deltas[i]), + axis=left_rank_dim) + else: + rank_1 = right_tangent_tt_ranks[i] + rank_2 = left_tangent_tt_ranks[i + 1] + if tt.is_tt_matrix(): + mode_size_n = raw_shape[0][i] + mode_size_m = raw_shape[1][i] + shape = [rank_1, mode_size_n, mode_size_m, rank_2] + else: + mode_size_n = raw_shape[0][i] + shape = [rank_1, mode_size_n, rank_2] + if is_batch_case: + shape = [batch_size] + shape + zeros = tf.zeros(shape, dtype=dtype) + upper = tf.concat((right_tt_core, zeros), axis=right_rank_dim) + lower = tf.concat((deltas[i], left_tt_core), axis=right_rank_dim) + tangent_core = tf.concat((upper, lower), axis=left_rank_dim) + cores.append(tangent_core) + if is_batch_case: + tangent = TensorTrainBatch(cores, batch_size=batch_size) + else: + tangent = TensorTrain(cores) + tangent.projection_on = tt + return tangent From eacac0465df6acff436e4c0aab1062f7810a0248 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 20:59:12 +0000 Subject: [PATCH 117/233] more linter errors: indentions --- t3f/riemannian_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/t3f/riemannian_test.py b/t3f/riemannian_test.py index 3055d831..22eee493 100644 --- a/t3f/riemannian_test.py +++ b/t3f/riemannian_test.py @@ -288,10 +288,10 @@ def testToAndFromDeltas(self): projected_normsq_actual = tf.add_n([tf.reduce_sum(c * c) for c in deltas]) with self.test_session() as sess: desired_val, actual_val = sess.run((ops.full(projected), - ops.full(reconstructed_projected))) + ops.full(reconstructed_projected))) self.assertAllClose(desired_val, actual_val) desired_val, actual_val = sess.run((projected_normsq_desired, - projected_normsq_actual)) + projected_normsq_actual)) self.assertAllClose(desired_val, actual_val) def testToAndFromDeltasBatch(self): @@ -312,10 +312,10 @@ def testToAndFromDeltasBatch(self): with self.test_session() as sess: desired_val, actual_val = sess.run((ops.full(projected), - ops.full(reconstructed_projected))) + ops.full(reconstructed_projected))) self.assertAllClose(desired_val, actual_val) desired_val, actual_val = sess.run((projected_normsq_desired, - projected_normsq_actual)) + projected_normsq_actual)) self.assertAllClose(desired_val, actual_val) From 07e018eb70789deda203a9e120dcbc44f60d9a3c Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:04:00 +0000 Subject: [PATCH 118/233] Simplify rank logic a bit --- t3f/autodiff.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index d4e69899..2d53d3ed 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -1,5 +1,6 @@ import tensorflow as tf +from t3f import shapes from t3f import decompositions from t3f import riemannian @@ -7,15 +8,13 @@ def _enforce_gauge_conditions(deltas, left): """Project deltas that define tangent space vec onto the gauge conditions.""" proj_deltas = [] + tt_ranks = shapes.lazy_tt_ranks(left) for i in range(left.ndims()): - if left.is_tt_matrix(): - r1, n, m, r2 = left.tt_cores[i].shape.as_list() - else: - r1, n, r2 = left.tt_cores[i].shape.as_list() - q = tf.reshape(left.tt_cores[i], (-1, r2)) + right_r = tt_ranks[i + 1] + q = tf.reshape(left.tt_cores[i], (-1, right_r)) if i < left.ndims() - 1: proj_delta = deltas[i] - proj_delta = tf.reshape(proj_delta, (-1, r2)) + proj_delta = tf.reshape(proj_delta, (-1, right_r)) proj_delta -= tf.matmul(q, tf.matmul(tf.transpose(q), proj_delta)) proj_delta = tf.reshape(proj_delta, left.tt_cores[i].shape) else: From 6ff7d08fe6c5430e4664df2eb58b461e6fc52992 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:08:39 +0000 Subject: [PATCH 119/233] s/debug/runtime_check/ --- t3f/autodiff.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 2d53d3ed..77206266 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -41,9 +41,10 @@ def _is_invariant_to_input_transforms(f_value_1, f_value_2, Returns: tf.op, assertion operation. """ - print('Warning: debug mode of Riemannian autodiff is turned on which ' - 'makes things a bit slower. It is advisable to keep debug=True untill ' - 'actuall production usage, since debug mode does help to catch bugs.') + print('Warning: runtime_check of Riemannian autodiff is turned on which ' + 'makes things a bit slower. It is advisable to keep runtime_check=True ' + 'untill actuall production usage, since runtime check does help to ' + 'catch bugs.') rel_diff = tf.abs((f_value_1 - f_value_2) / f_value_1) err_msg = "The function passed to Riemannian autodiff returns different " \ "values for two different versions of the same tensor. " \ @@ -53,7 +54,7 @@ def _is_invariant_to_input_transforms(f_value_1, f_value_2, return assert_op -def gradients(func, x, name='t3f_gradients', debug=True): +def gradients(func, x, name='t3f_gradients', runtime_check=True): """Riemannian autodiff: returns gradient projected on tangent space of TT. Computes projection of the gradient df/dx onto the tangent space of TT tensor @@ -77,10 +78,10 @@ def gradients(func, x, name='t3f_gradients', debug=True): x: point at which to compute the gradient and on which tangent space to project the gradient. name: string, name of the Op. - debug: [True] whether to do a sanity check that the passed function is - invariant to different TT representations (otherwise the Rieamnnian - gradient doesn't even exist). It makes things slower, but helps catching - bugs, so turn it off during production deployment. + runtime_check: [True] whether to do a sanity check that the passed + function is invariant to different TT representations (otherwise + the Rieamnnian gradient doesn't even exist). It makes things slower, + but helps catching bugs, so turn it off during production deployment. Returns: `TensorTrain`, projection of the gradient df/dx onto the tangent space at @@ -96,7 +97,7 @@ def gradients(func, x, name='t3f_gradients', debug=True): deltas += [tf.zeros_like(cc) for cc in right.tt_cores[1:]] x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) function_value = func(x_projection) - if debug: + if runtime_check: assert_op = _is_invariant_to_input_transforms(function_value, func(right)) else: assert_op = None @@ -107,7 +108,7 @@ def gradients(func, x, name='t3f_gradients', debug=True): def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', - debug=True): + runtime_check=True): """P_x d^2f/dx^2 P_x vector, i.e. Riemannian hessian by vector product. Computes @@ -138,10 +139,10 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', project the gradient. vector: `TensorTrain` object which to multiply be the Hessian. name: string, name of the Op. - debug: [True] whether to do a sanity check that the passed function is - invariant to different TT representations (otherwise the Rieamnnian - gradient doesn't even exist). It makes things slower, but helps catching - bugs, so turn it off during production deployment. + runtime_check: [True] whether to do a sanity check that the passed + function is invariant to different TT representations (otherwise + the Rieamnnian gradient doesn't even exist). It makes things slower, + but helps catching bugs, so turn it off during production deployment. Returns: `TensorTrain`, projection of the gradient df/dx onto the tangent space at @@ -158,7 +159,7 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', deltas += [tf.zeros_like(cc) for cc in right.tt_cores[1:]] x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) function_value = func(x_projection) - if debug: + if runtime_check: assert_op = _is_invariant_to_input_transforms(function_value, func(right)) else: assert_op = None From fd4049b9d6a5cefec107c9f7c9a8e51227050340 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:15:13 +0000 Subject: [PATCH 120/233] Add all input tensors to tf.name_scope dependencies --- t3f/riemannian.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/t3f/riemannian.py b/t3f/riemannian.py index 8322dcdb..5a92f5a7 100644 --- a/t3f/riemannian.py +++ b/t3f/riemannian.py @@ -827,7 +827,12 @@ def deltas_to_tangent_space(deltas, tt, left=None, right=None, dtype = tt.dtype num_dims = tt.ndims() # TODO: add cache instead of mannually pasisng precomputed stuff? - with tf.name_scope(name, values=tt.tt_cores): + input_tensors = list(tt.tt_cores) + list(deltas) + if left is not None: + input_tensors += list(left.tt_cores) + if right is not None: + input_tensors += list(right.tt_cores) + with tf.name_scope(name, values=input_tensors): if left is None: left = decompositions.orthogonalize_tt_cores(tt) if right is None: From dc17feacdd8979a1b7f63def7454c0dca914ad7f Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:16:13 +0000 Subject: [PATCH 121/233] Compare func(x_augmentd) against func(x) instead of against func(right) because its a bit faster and a better test since even the ranks are different --- t3f/autodiff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 77206266..96fe68d8 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -98,7 +98,7 @@ def gradients(func, x, name='t3f_gradients', runtime_check=True): x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) function_value = func(x_projection) if runtime_check: - assert_op = _is_invariant_to_input_transforms(function_value, func(right)) + assert_op = _is_invariant_to_input_transforms(function_value, func(x)) else: assert_op = None with tf.control_dependencies([assert_op]): @@ -160,7 +160,7 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right) function_value = func(x_projection) if runtime_check: - assert_op = _is_invariant_to_input_transforms(function_value, func(right)) + assert_op = _is_invariant_to_input_transforms(function_value, func(x)) else: assert_op = None with tf.control_dependencies([assert_op]): From 5e6dca302e31859dff593d9ac77600ba819d5432 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:18:18 +0000 Subject: [PATCH 122/233] More precise linearised hessian formula --- t3f/autodiff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 96fe68d8..f7bf96af 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -109,10 +109,10 @@ def gradients(func, x, name='t3f_gradients', runtime_check=True): def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', runtime_check=True): - """P_x d^2f/dx^2 P_x vector, i.e. Riemannian hessian by vector product. + """P_x [d^2f/dx^2] P_x vector, i.e. Riemannian hessian by vector product. Computes - P_x d^2f/dx^2 P_x vector + P_x [d^2f/dx^2] P_x vector where P_x is projection onto the tangent space of TT at point x and d^2f/dx^2 is the Hessian of the function. From d862d014071b26622c0f017355d95d9b90b2d0ed Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:20:26 +0000 Subject: [PATCH 123/233] Add forgotten projection in the example --- t3f/autodiff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index f7bf96af..63f78a37 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -129,7 +129,8 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', # Quadratic form with matrix A: . # It's gradient is (A + A.T) x, it's Hessian is (A + A.T) # It's Riemannian Hessian by vector product is - # t3f.project(t3f.matmul(A + t3f.transpose(A), vector), x) + # proj_vec = t3f.project(vector, x) + # t3f.project(t3f.matmul(A + t3f.transpose(A), proj_vec), x) f = lambda x: t3f.quadratic_form(A, x, x) res = t3f.hessian_vector_product(f, x, vector) From 1187db5be2185069245a72dc3205b39eb514cb52 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:21:43 +0000 Subject: [PATCH 124/233] Fix copypasted comment --- t3f/autodiff.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 63f78a37..96bd9aac 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -146,8 +146,7 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', but helps catching bugs, so turn it off during production deployment. Returns: - `TensorTrain`, projection of the gradient df/dx onto the tangent space at - point x. + `TensorTrain`, result of the Riemannian hessian by vector product. See also: t3f.gradients From a0e87900248cc89a60f7e98264ce60a4e9e8b764 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:26:46 +0000 Subject: [PATCH 125/233] Remove unused import --- t3f/autodiff_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index 6903998b..4f3c8600 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -3,7 +3,6 @@ from t3f import ops from t3f import initializers -from t3f import variables from t3f import riemannian from t3f import autodiff @@ -96,4 +95,3 @@ class AutodiffTestFloat64(tf.test.TestCase, _AutodiffTest): if __name__ == "__main__": tf.test.main() - From af70f39236452681255548b50033b67d28d64015 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:31:33 +0000 Subject: [PATCH 126/233] compare(desired, actual) instead of (actual, desired) --- t3f/autodiff_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index 4f3c8600..e6a58da9 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -21,8 +21,8 @@ def func1(x): actual1 = ops.full(autodiff.gradients(func1, x)) desired1 = ops.full(ops.flat_inner(x, w) * riemannian.project(w, x)) with self.test_session() as sess: - actual1_v, desired1_v = sess.run([actual1, desired1]) - np.testing.assert_allclose(actual1_v, desired1_v, rtol=1e-4) + desired1_v, actual1_v = sess.run([desired1, actual1]) + np.testing.assert_allclose(desired1_v, actual1_v, rtol=1e-4) def func2(x): return ops.quadratic_form(A, x, x) @@ -31,8 +31,8 @@ def func2(x): grad = ops.matmul(ops.transpose(A) + A, x) desired2 = ops.full(riemannian.project(grad, x)) with self.test_session() as sess: - actual_v2, desired_v2 = sess.run([actual2, desired2]) - np.testing.assert_allclose(actual_v2, desired_v2, rtol=1e-4) + desired2_v, actual2_v = sess.run([desired2, actual2]) + np.testing.assert_allclose(desired2_v, actual2_v, rtol=1e-4) def func3(x): # A function which is not invariant to different representations of the @@ -61,8 +61,8 @@ def func1(x): desired1 = riemannian.project(ops.flat_inner(projected_vector, w) * w, x) desired1 = ops.full(desired1) with self.test_session() as sess: - actual1_v, desired1_v = sess.run([actual1, desired1]) - np.testing.assert_allclose(actual1_v, desired1_v, rtol=1e-4) + desired1_v, actual1_v = sess.run([desired1, actual1]) + np.testing.assert_allclose(desired1_v, actual1_v, rtol=1e-4) def func2(x): return ops.quadratic_form(A, x, x) @@ -71,8 +71,8 @@ def func2(x): hessian_by_vector = ops.matmul(ops.transpose(A) + A, projected_vector) desired2 = ops.full(riemannian.project(hessian_by_vector, x)) with self.test_session() as sess: - actual2_v, desired2_v = sess.run([actual2, desired2]) - np.testing.assert_allclose(actual2_v, desired2_v, rtol=1e-3) + desired2_v, actual2_v = sess.run([desired2, actual2]) + np.testing.assert_allclose(desired2_v, actual2_v, rtol=1e-4) def func3(x): # A function which is not invariant to different representations of the From 96bc4f50fd0a942eb1e9b55579eefc37bace4978 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 7 Nov 2018 21:44:23 +0000 Subject: [PATCH 127/233] remove unused variable --- t3f/autodiff_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index e6a58da9..95eda504 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -13,7 +13,6 @@ def testGradients(self): w = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) A = initializers.random_matrix(([5] * 3, [5] * 3), dtype=self.dtype) x = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) - z = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) def func1(x): return 0.5 * ops.flat_inner(x, w) ** 2 From ffa0f3828246431cc751657df5d17b85cf36d2e0 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 11 Nov 2018 14:23:26 +0000 Subject: [PATCH 128/233] typo fix --- t3f/batch_ops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t3f/batch_ops.py b/t3f/batch_ops.py index ca230ff7..e8451b76 100644 --- a/t3f/batch_ops.py +++ b/t3f/batch_ops.py @@ -176,9 +176,9 @@ def pairwise_flat_inner(tt_1, tt_2, matrix=None, res = tf.einsum(einsum_str, res, curr_core_1, curr_core_2) else: # res[i, j] = tt_1[i] ^ T * matrix * tt_2[j] - are_all_maatrices = tt_1.is_tt_matrix() and tt_2.is_tt_matrix() - are_all_maatrices = are_all_maatrices and matrix.is_tt_matrix() - if not are_all_maatrices: + are_all_matrices = tt_1.is_tt_matrix() and tt_2.is_tt_matrix() + are_all_matrices = are_all_matrices and matrix.is_tt_matrix() + if not are_all_matrices: raise ValueError('When passing three arguments to pairwise_flat_inner, ' 'the first 2 of them should be TT-vecors and the last ' 'should be a TT-matrix. Got %s, %s, and %s instead.' % From 999ef8d2c18c8cb214c55d21683dfe92c1f6845e Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 11 Nov 2018 14:24:20 +0000 Subject: [PATCH 129/233] actually use name arg of decompositions --- t3f/decompositions.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/t3f/decompositions.py b/t3f/decompositions.py index b3ee16dd..b02dcf3b 100644 --- a/t3f/decompositions.py +++ b/t3f/decompositions.py @@ -233,10 +233,12 @@ def round(tt, max_tt_rank=None, epsilon=None, name='t3f_round'): not a vector of length d + 1 where d is the number of dimensions (rank) of the input tensor, if epsilon is less than 0. """ - if isinstance(tt, TensorTrainBatch): - return _round_batch_tt(tt, max_tt_rank, epsilon) - else: - return _round_tt(tt, max_tt_rank, epsilon) + # TODO: add epsilon to the name_scope dependencies. + with tf.name_scope(name, values=tt.tt_cores): + if isinstance(tt, TensorTrainBatch): + return _round_batch_tt(tt, max_tt_rank, epsilon) + else: + return _round_tt(tt, max_tt_rank, epsilon) def _round_tt(tt, max_tt_rank, epsilon): From a0fb7c9b4c207c0f697b9258d81fca645767f746 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 11 Nov 2018 14:28:23 +0000 Subject: [PATCH 130/233] do not pass tt as dependency for zeros_like --- t3f/initializers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/t3f/initializers.py b/t3f/initializers.py index ebf65a26..f7a60cc0 100644 --- a/t3f/initializers.py +++ b/t3f/initializers.py @@ -451,7 +451,10 @@ def ones_like(tt, name='t3f_ones_like'): raise ValueError("`tt` has to be a Tensor Train object") else: shape = shapes.lazy_raw_shape(tt) - with tf.name_scope(name, values=tt.tt_cores): + # I guess variables=tt.tt_cores is not needed here since the output of + # the function doesn't depend on the values of the TT-cores, only on their + # shapes etc. But I'm not 100% sure. + with tf.name_scope(name): if tt.is_tt_matrix(): return matrix_ones(shape, dtype=tt.dtype) else: @@ -477,7 +480,10 @@ def zeros_like(tt, name='t3f_zeros_like'): raise ValueError("`tt` has to be a Tensor Train object") else: shape = shapes.lazy_raw_shape(tt) - with tf.name_scope(name, values=tt.tt_cores): + # I guess variables=tt.tt_cores is not needed here since the output of + # the function doesn't depend on the values of the TT-cores, only on their + # shapes etc. But I'm not 100% sure. + with tf.name_scope(name): if tt.is_tt_matrix(): return matrix_zeros(shape, dtype=tt.dtype) else: From f078dad9ca6329420cd410c1537c24984d37eb5a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 11 Nov 2018 14:32:32 +0000 Subject: [PATCH 131/233] support name arg in kronecker module --- t3f/kronecker.py | 161 ++++++++++++++++++++++++----------------------- 1 file changed, 84 insertions(+), 77 deletions(-) diff --git a/t3f/kronecker.py b/t3f/kronecker.py index 0910c364..6281dec2 100644 --- a/t3f/kronecker.py +++ b/t3f/kronecker.py @@ -5,15 +5,16 @@ from t3f import ops -def determinant(kron_a): +def determinant(kron_a, name='t3f_kronecker_determinant'): """Computes the determinant of a given Kronecker-factorized matrix. Note, that this method can suffer from overflow. Args: kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a - batch of matrices of size N x N, factorized into a Kronecker product of - square matrices (all tt-ranks are 1 and all tt-cores are square). + batch of matrices of size N x N, factorized into a Kronecker product of + square matrices (all tt-ranks are 1 and all tt-cores are square). + name: string, name of the Op. Returns: A number or a Tensor with numbers for each element in the batch. @@ -41,28 +42,30 @@ def determinant(kron_a): 'matrices (tt-cores must be square)') is_batch = isinstance(kron_a, TensorTrainBatch) - pows = tf.cast(tf.reduce_prod(i_shapes), kron_a.dtype) - cores = kron_a.tt_cores - det = 1 - for core_idx in range(kron_a.ndims()): - core = cores[core_idx] - if is_batch: - core_det = tf.matrix_determinant(core[:, 0, :, :, 0]) - else: - core_det = tf.matrix_determinant(core[0, :, :, 0]) - core_pow = pows / i_shapes[core_idx].value - - det *= tf.pow(core_det, core_pow) - return det - - -def slog_determinant(kron_a): + with tf.name_scope(name, values=kron_a.tt_cores): + pows = tf.cast(tf.reduce_prod(i_shapes), kron_a.dtype) + cores = kron_a.tt_cores + det = 1 + for core_idx in range(kron_a.ndims()): + core = cores[core_idx] + if is_batch: + core_det = tf.matrix_determinant(core[:, 0, :, :, 0]) + else: + core_det = tf.matrix_determinant(core[0, :, :, 0]) + core_pow = pows / i_shapes[core_idx].value + + det *= tf.pow(core_det, core_pow) + return det + + +def slog_determinant(kron_a, name='t3f_kronecker_slog_determinant'): """Computes the sign and log-det of a given Kronecker-factorized matrix. Args: kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a - batch of matrices of size N x N, factorized into a Kronecker product of - square matrices (all tt-ranks are 1 and all tt-cores are square). + batch of matrices of size N x N, factorized into a Kronecker product of + square matrices (all tt-ranks are 1 and all tt-cores are square). + name: string, name of the Op. Returns: Two number or two Tensor with numbers for each element in the batch. @@ -92,31 +95,33 @@ def slog_determinant(kron_a): 'matrices (tt-cores must be square)') is_batch = isinstance(kron_a, TensorTrainBatch) - pows = tf.cast(tf.reduce_prod(i_shapes), kron_a.dtype) - logdet = 0. - det_sign = 1. - - for core_idx in range(kron_a.ndims()): - core = kron_a.tt_cores[core_idx] - if is_batch: - core_det = tf.matrix_determinant(core[:, 0, :, :, 0]) - else: - core_det = tf.matrix_determinant(core[0, :, :, 0]) - core_abs_det = tf.abs(core_det) - core_det_sign = tf.sign(core_det) - core_pow = pows / i_shapes[core_idx].value - logdet += tf.log(core_abs_det) * core_pow - det_sign *= core_det_sign**(core_pow) - return det_sign, logdet - - -def inv(kron_a): + with tf.name_scope(name, values=kron_a.tt_cores): + pows = tf.cast(tf.reduce_prod(i_shapes), kron_a.dtype) + logdet = 0. + det_sign = 1. + + for core_idx in range(kron_a.ndims()): + core = kron_a.tt_cores[core_idx] + if is_batch: + core_det = tf.matrix_determinant(core[:, 0, :, :, 0]) + else: + core_det = tf.matrix_determinant(core[0, :, :, 0]) + core_abs_det = tf.abs(core_det) + core_det_sign = tf.sign(core_det) + core_pow = pows / i_shapes[core_idx].value + logdet += tf.log(core_abs_det) * core_pow + det_sign *= core_det_sign**(core_pow) + return det_sign, logdet + + +def inv(kron_a, name='t3f_kronecker_inv'): """Computes the inverse of a given Kronecker-factorized matrix. Args: kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a - batch of matrices of size N x N, factorized into a Kronecker product of - square matrices (all tt-ranks are 1 and all tt-cores are square). + batch of matrices of size N x N, factorized into a Kronecker product of + square matrices (all tt-ranks are 1 and all tt-cores are square). + name: string, name of the Op. Returns: `TensorTrain` object containing a TT-matrix of size N x N if the argument is @@ -146,33 +151,35 @@ def inv(kron_a): 'matrices (tt-cores must be square)') is_batch = isinstance(kron_a, TensorTrainBatch) - inv_cores = [] - for core_idx in range(kron_a.ndims()): - core = kron_a.tt_cores[core_idx] + with tf.name_scope(name, values=kron_a.tt_cores): + inv_cores = [] + for core_idx in range(kron_a.ndims()): + core = kron_a.tt_cores[core_idx] + if is_batch: + core_inv = tf.matrix_inverse(core[:, 0, :, :, 0]) + core_inv = tf.expand_dims(tf.expand_dims(core_inv, 1), -1) + else: + core_inv = tf.matrix_inverse(core[0, :, :, 0]) + core_inv = tf.expand_dims(tf.expand_dims(core_inv, 0), -1) + inv_cores.append(core_inv) + + res_ranks = kron_a.get_tt_ranks() + res_shape = kron_a.get_raw_shape() if is_batch: - core_inv = tf.matrix_inverse(core[:, 0, :, :, 0]) - core_inv = tf.expand_dims(tf.expand_dims(core_inv, 1), -1) + return TensorTrainBatch(inv_cores, res_shape, res_ranks) else: - core_inv = tf.matrix_inverse(core[0, :, :, 0]) - core_inv = tf.expand_dims(tf.expand_dims(core_inv, 0), -1) - inv_cores.append(core_inv) - - res_ranks = kron_a.get_tt_ranks() - res_shape = kron_a.get_raw_shape() - if is_batch: - return TensorTrainBatch(inv_cores, res_shape, res_ranks) - else: - return TensorTrain(inv_cores, res_shape, res_ranks) + return TensorTrain(inv_cores, res_shape, res_ranks) -def cholesky(kron_a): +def cholesky(kron_a, name='t3f_kronecker_cholesky'): """Computes the Cholesky decomposition of a given Kronecker-factorized matrix. Args: kron_a: `TensorTrain` or `TensorTrainBatch` object containing a matrix or a - batch of matrices of size N x N, factorized into a Kronecker product of - square matrices (all tt-ranks are 1 and all tt-cores are square). All the - cores must be symmetric positive-definite. + batch of matrices of size N x N, factorized into a Kronecker product of + square matrices (all tt-ranks are 1 and all tt-cores are square). All the + cores must be symmetric positive-definite. + name: string, name of the Op. Returns: `TensorTrain` object containing a TT-matrix of size N x N if the argument is @@ -202,24 +209,24 @@ def cholesky(kron_a): 'matrices (tt-cores must be square)') is_batch = isinstance(kron_a, TensorTrainBatch) - cho_cores = [] - - for core_idx in range(kron_a.ndims()): - core = kron_a.tt_cores[core_idx] + with tf.name_scope(name, values=kron_a.tt_cores): + cho_cores = [] + for core_idx in range(kron_a.ndims()): + core = kron_a.tt_cores[core_idx] + if is_batch: + core_cho = tf.cholesky(core[:, 0, :, :, 0]) + core_cho = tf.expand_dims(tf.expand_dims(core_cho, 1), -1) + else: + core_cho = tf.cholesky(core[0, :, :, 0]) + core_cho = tf.expand_dims(tf.expand_dims(core_cho, 0), -1) + cho_cores.append(core_cho) + + res_ranks = kron_a.get_tt_ranks() + res_shape = kron_a.get_raw_shape() if is_batch: - core_cho = tf.cholesky(core[:, 0, :, :, 0]) - core_cho = tf.expand_dims(tf.expand_dims(core_cho, 1), -1) + return TensorTrainBatch(cho_cores, res_shape, res_ranks) else: - core_cho = tf.cholesky(core[0, :, :, 0]) - core_cho = tf.expand_dims(tf.expand_dims(core_cho, 0), -1) - cho_cores.append(core_cho) - - res_ranks = kron_a.get_tt_ranks() - res_shape = kron_a.get_raw_shape() - if is_batch: - return TensorTrainBatch(cho_cores, res_shape, res_ranks) - else: - return TensorTrain(cho_cores, res_shape, res_ranks) + return TensorTrain(cho_cores, res_shape, res_ranks) def _is_kron(tt_a): From 2e59136cd1d4c6cccc32efabbdcb74f891d451c7 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 11 Nov 2018 14:33:10 +0000 Subject: [PATCH 132/233] fix broken name_scope variables argument use --- t3f/regularizers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/regularizers.py b/t3f/regularizers.py index 764d2f74..8144ab1f 100644 --- a/t3f/regularizers.py +++ b/t3f/regularizers.py @@ -30,7 +30,7 @@ def l2_regularizer(scale, scope=None): def l2(tt): """Applies l2 regularization to TensorTrain object.""" - with tf.name_scope(scope, 'l2_regularizer', [tt]) as name: + with tf.name_scope(scope, 'l2_regularizer', values=tt.tt_cores) as name: my_scale = tf.convert_to_tensor(scale, dtype=tt.dtype, name='scale') return tf.multiply(my_scale, ops.frobenius_norm_squared(tt), name=name) @@ -65,7 +65,7 @@ def cores_regularizer(core_regularizer, scale, scope=None): def regularizer(tt): """Applies the regularization to TensorTrain object.""" - with tf.name_scope(scope, 'l2_regularizer', [tt]) as name: + with tf.name_scope(scope, 'l2_regularizer', values=tt.tt_cores) as name: my_scale = tf.convert_to_tensor(scale, dtype=tt.dtype, name='scale') penalty = 0.0 for i in range(tt.ndims()): From b9368bde1af21efc186e5049834e3489642839e4 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 11 Nov 2018 14:35:28 +0000 Subject: [PATCH 133/233] support name arg in shapes module --- t3f/shapes.py | 207 ++++++++++++++++++++++++++++---------------------- 1 file changed, 117 insertions(+), 90 deletions(-) diff --git a/t3f/shapes.py b/t3f/shapes.py index bc411092..8b65f12b 100644 --- a/t3f/shapes.py +++ b/t3f/shapes.py @@ -3,7 +3,7 @@ # TODO: test all these functions. -def tt_ranks(tt): +def tt_ranks(tt, name='t3f_tt_ranks'): """Returns the TT-ranks of a TensorTrain. This operation returns a 1-D integer tensor representing the TT-ranks of @@ -11,19 +11,21 @@ def tt_ranks(tt): Args: tt: `TensorTrain` or `TensorTrainBatch` object. + name: string, name of the Op. Returns: A `Tensor` """ num_dims = tt.ndims() ranks = [] - for i in range(num_dims): - ranks.append(tf.shape(tt.tt_cores[i])[tt.left_tt_rank_dim]) - ranks.append(tf.shape(tt.tt_cores[-1])[-1]) - return tf.stack(ranks, axis=0) + with tf.name_scope(name, values=tt.tt_cores): + for i in range(num_dims): + ranks.append(tf.shape(tt.tt_cores[i])[tt.left_tt_rank_dim]) + ranks.append(tf.shape(tt.tt_cores[-1])[-1]) + return tf.stack(ranks, axis=0) -def shape(tt): +def shape(tt, name='t3f_shape'): """Returns the shape of a TensorTrain. This operation returns a 1-D integer tensor representing the shape of @@ -34,25 +36,27 @@ def shape(tt): Args: tt: `TensorTrain` or `TensorTrainBatch` object. + name: string, name of the Op. Returns: A `Tensor` """ - tt_raw_shape = raw_shape(tt) - if tt.is_tt_matrix(): - res = tf.reduce_prod(tt_raw_shape, axis=1) - else: - res = tt_raw_shape[0] + with tf.name_scope(name, values=tt.tt_cores): + tt_raw_shape = raw_shape(tt) + if tt.is_tt_matrix(): + res = tf.reduce_prod(tt_raw_shape, axis=1) + else: + res = tt_raw_shape[0] - # TODO: ugly. - from t3f.tensor_train_batch import TensorTrainBatch - if isinstance(tt, TensorTrainBatch): - res = tf.concat((tf.expand_dims(batch_size(tt), 0), res), axis=0) + # TODO: ugly. + from t3f.tensor_train_batch import TensorTrainBatch + if isinstance(tt, TensorTrainBatch): + res = tf.concat((tf.expand_dims(batch_size(tt), 0), res), axis=0) - return res + return res -def raw_shape(tt): +def raw_shape(tt, name='t3f_raw_shape'): """Returns the shape of a TensorTrain. This operation returns a 2-D integer tensor representing the shape of @@ -63,6 +67,7 @@ def raw_shape(tt): Args: tt: `TensorTrain` or `TensorTrainBatch` object. + name: string, name of the Op. Returns: A 2-D `Tensor` of size 1 x ndims() or 2 x ndims() @@ -70,32 +75,40 @@ def raw_shape(tt): num_dims = tt.ndims() num_tensor_axis = len(tt.get_raw_shape()) final_raw_shape = [] - # TODO: ugly. - from t3f.tensor_train import TensorTrain - axes_shift = 1 if isinstance(tt, TensorTrain) else 2 - for ax in range(num_tensor_axis): - curr_raw_shape = [] - for core_idx in range(num_dims): - curr_raw_shape.append(tf.shape(tt.tt_cores[core_idx])[ax + axes_shift]) - final_raw_shape.append(tf.stack(curr_raw_shape, axis=0)) - return tf.stack(final_raw_shape, axis=0) - - -def batch_size(tt): + with tf.name_scope(name, values=tt.tt_cores): + # TODO: ugly. + from t3f.tensor_train import TensorTrain + axes_shift = 1 if isinstance(tt, TensorTrain) else 2 + for ax in range(num_tensor_axis): + curr_raw_shape = [] + for core_idx in range(num_dims): + curr_raw_shape.append(tf.shape(tt.tt_cores[core_idx])[ax + axes_shift]) + final_raw_shape.append(tf.stack(curr_raw_shape, axis=0)) + return tf.stack(final_raw_shape, axis=0) + + +def batch_size(tt, name='t3f_batch_size'): """Return the number of elements in a TensorTrainBatch. + + Args: + tt: `TensorTrainBatch` object. + name: string, name of the Op. - Return 0-D integer tensor. + Returns: + 0-D integer tensor. Raises: - ValueError if got `TensorTrain` which doesn't have batch_size as input.""" + ValueError if got `TensorTrain` which doesn't have batch_size as input. + """ if not hasattr(tt, 'batch_size'): raise ValueError('batch size is not available for a TensorTrain object.') first_core = tt.tt_cores[0] # The first dimension of any TT-core in TensorTrainBatch is the batch size. - return tf.shape(first_core)[0] + with tf.name_scope(name, values=tt.tt_cores): + return tf.shape(first_core)[0] -def lazy_tt_ranks(tt): +def lazy_tt_ranks(tt, name='t3f_lazy_tt_ranks'): """Returns static TT-ranks of a TensorTrain if defined, and dynamic otherwise. This operation returns a 1-D integer numpy array of TT-ranks if they are @@ -104,18 +117,20 @@ def lazy_tt_ranks(tt): Args: tt: `TensorTrain` object. + name: string, name of the Op. Returns: A 1-D numpy array or `tf.Tensor` """ - static_tt_ranks = tt.get_tt_ranks() - if static_tt_ranks.is_fully_defined(): - return np.array(static_tt_ranks.as_list()) - else: - return tt_ranks(tt) + with tf.name_scope(name, values=tt.tt_cores): + static_tt_ranks = tt.get_tt_ranks() + if static_tt_ranks.is_fully_defined(): + return np.array(static_tt_ranks.as_list()) + else: + return tt_ranks(tt) -def lazy_shape(tt): +def lazy_shape(tt, name='t3f_lazy_shape'): """Returns static shape of a TensorTrain if defined, and dynamic otherwise. This operation returns a 1-D integer numpy array representing the shape of the @@ -124,18 +139,20 @@ def lazy_shape(tt): Args: tt: `TensorTrain` object. + name: string, name of the Op. Returns: A 1-D numpy array or `tf.Tensor` """ - static_shape = tt.get_shape() - if static_shape.is_fully_defined(): - return np.array(static_shape.as_list()) - else: - return shape(tt) + with tf.name_scope(name, values=tt.tt_cores): + static_shape = tt.get_shape() + if static_shape.is_fully_defined(): + return np.array(static_shape.as_list()) + else: + return shape(tt) -def lazy_raw_shape(tt): +def lazy_raw_shape(tt, name='t3f_lazy_raw_shape'): """Returns static raw shape of a TensorTrain if defined, and dynamic otherwise. This operation returns a 2-D integer numpy array representing the raw shape of @@ -147,23 +164,26 @@ def lazy_raw_shape(tt): Args: tt: `TensorTrain` object. + name: string, name of the Op. Returns: A 2-D numpy array or `tf.Tensor` of size 1 x ndims() or 2 x ndims() """ # If get_shape is fully defined, it guaranties that all elements of raw shape # are defined. - if tt.get_shape().is_fully_defined(): - return np.array([s.as_list() for s in tt.get_raw_shape()]) - else: - return raw_shape(tt) + with tf.name_scope(name, values=tt.tt_cores): + if tt.get_shape().is_fully_defined(): + return np.array([s.as_list() for s in tt.get_raw_shape()]) + else: + return raw_shape(tt) -def lazy_batch_size(tt): +def lazy_batch_size(tt, name='t3f_lazy_batch_size'): """Return static batch_size if available and dynamic otherwise. Args: tt: `TensorTrainBatch` object. + name: string, name of the Op. Returns: A number or a 0-D `tf.Tensor` @@ -172,40 +192,43 @@ def lazy_batch_size(tt): ValueError if got `TensorTrain` which doesn't have batch_size as input.""" if not hasattr(tt, 'batch_size'): raise ValueError('batch size is not available for a TensorTrain object.') - if tt.batch_size is not None: - return tt.batch_size - else: - return batch_size(tt) + with tf.name_scope(name, values=tt.tt_cores): + if tt.batch_size is not None: + return tt.batch_size + else: + return batch_size(tt) -def clean_raw_shape(shape): +def clean_raw_shape(shape, name='t3f_clean_raw_shape'): """Returns a tuple of TensorShapes for any valid shape representation. Args: shape: An np.array, a tf.TensorShape (for tensors), a tuple of tf.TensorShapes (for TT-matrices or tensors), or None + name: string, name of the Op. Returns: A tuple of tf.TensorShape, or None if the input is None """ if shape is None: return None - if isinstance(shape, tf.TensorShape) or isinstance(shape[0], tf.TensorShape): - # Assume tf.TensorShape. - if isinstance(shape, tf.TensorShape): - shape = tuple((shape,)) - else: - np_shape = np.array(shape) - # Make sure that the shape is 2-d array both for tensors and TT-matrices. - np_shape = np.squeeze(np_shape) - if len(np_shape.shape) == 1: - # A tensor. - np_shape = [np_shape] - shape = [] - for i in range(len(np_shape)): - shape.append(tf.TensorShape(np_shape[i])) - shape = tuple(shape) - return shape + with tf.name_scope(name, values=shape): + if isinstance(shape, tf.TensorShape) or isinstance(shape[0], tf.TensorShape): + # Assume tf.TensorShape. + if isinstance(shape, tf.TensorShape): + shape = tuple((shape,)) + else: + np_shape = np.array(shape) + # Make sure that the shape is 2-d array both for tensors and TT-matrices. + np_shape = np.squeeze(np_shape) + if len(np_shape.shape) == 1: + # A tensor. + np_shape = [np_shape] + shape = [] + for i in range(len(np_shape)): + shape.append(tf.TensorShape(np_shape[i])) + shape = tuple(shape) + return shape def is_batch_broadcasting_possible(tt_a, tt_b): @@ -239,43 +262,47 @@ def is_batch_broadcasting_possible(tt_a, tt_b): return True -def squeeze_batch_dim(tt): +def squeeze_batch_dim(tt, name='t3f_squeeze_batch_dim'): """Converts batch size 1 TensorTrainBatch into TensorTrain. Args: tt: TensorTrain or TensorTrainBatch. + name: string, name of the Op. Returns: TensorTrain if the input is a TensorTrainBatch with batch_size == 1 (known at compilation stage) or a TensorTrain. TensorTrainBatch otherwise. """ - try: - if tt.batch_size == 1: - return tt[0] - else: + with tf.name_scope(name, values=tt.tt_cores): + try: + if tt.batch_size == 1: + return tt[0] + else: + return tt + except AttributeError: + # tt object does not have attribute batch_size, probably already + # a TensorTrain. return tt - except AttributeError: - # tt object does not have attribute batch_size, probably already - # a TensorTrain. - return tt -def expand_batch_dim(tt): +def expand_batch_dim(tt, name='t3f_expand_batch_dim'): """Creates a 1-element TensorTrainBatch from a TensorTrain. Args: tt: TensorTrain or TensorTrainBatch. + name: string, name of the Op. Returns: TensorTrainBatch """ - if hasattr(tt, 'batch_size'): - return tt - else: - from t3f.tensor_train_batch import TensorTrainBatch - tt_cores = [] - for core_idx in range(tt.ndims()): - tt_cores.append(tf.expand_dims(tt.tt_cores[core_idx], 0)) - return TensorTrainBatch(tt_cores, tt.get_raw_shape(), tt.get_tt_ranks(), - batch_size=1) + with tf.name_scope(name, values=tt.tt_cores): + if hasattr(tt, 'batch_size'): + return tt + else: + from t3f.tensor_train_batch import TensorTrainBatch + tt_cores = [] + for core_idx in range(tt.ndims()): + tt_cores.append(tf.expand_dims(tt.tt_cores[core_idx], 0)) + return TensorTrainBatch(tt_cores, tt.get_raw_shape(), tt.get_tt_ranks(), + batch_size=1) From 6a6f239be1db804ff7d724622b36e0e193d7d2ed Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 17:10:46 +0000 Subject: [PATCH 134/233] tf versions and latest version install --- docs/installation.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 3f3657ad..5229a79a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,18 +2,27 @@ sphinx-quickstart on Sun Mar 12 10:06:09 2017. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. +.. _InstallationInstructions: Installation ============ -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation, tested versions are from 1.0 to 1.11 (see here_ for TF installation instructions). +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation of version >= 1.2 (tested versions are from 1.2 to 1.11, see here_ for TF installation instructions). .. _here: https://www.tensorflow.org/install/ -We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. +We don't include TF into pip requirements since the installation of TensorFlow varies depending on your setup. -Then simply run +Then, to install the stable version, run .. code-block:: bash pip install t3f + +To install the latest version, run + +.. code-block:: bash + + git clone https://github.com/Bihaqo/t3f.git + cd t3f + pip install . \ No newline at end of file From ee5a02b0aa65a3eca6d659bc5f33f2ef93f7123c Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 17:15:10 +0000 Subject: [PATCH 135/233] init jypiter version of quick start --- docs/conf.py | 3 +- docs/quick_start.ipynb | 141 +++++++++++++++++++++++++++++++++++++++++ docs/quick_start.rst | 66 ------------------- 3 files changed, 143 insertions(+), 67 deletions(-) create mode 100644 docs/quick_start.ipynb delete mode 100644 docs/quick_start.rst diff --git a/docs/conf.py b/docs/conf.py index fc9d3031..5047eb36 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,7 +39,8 @@ 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', - 'sphinx.ext.mathjax'] + 'sphinx.ext.mathjax', + 'nbsphinx'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/quick_start.ipynb b/docs/quick_start.ipynb new file mode 100644 index 00000000..b62a362b --- /dev/null +++ b/docs/quick_start.ipynb @@ -0,0 +1,141 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Quick start\n", + "In this quick starting guide we show the basics of working with t3f library. The main concept of the library is a TensorTrain object -- a compact (factorized) representation of a tensor (=multidimensional array). This is generalization of the matrix low-rank decomposition.\n", + "\n", + "\n", + "To begin, install t3f (InstallationInstructions_), import some libraries, and enable [eager execution mode](https://www.tensorflow.org/guide/eager) which simplifies workflow with TensorFlow" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: check if T3F is installed and install it!\n", + "\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "import t3f\n", + "tf.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start with converting a dense (numpy) matrix into the TT-format, which in this case coinsides with the low-rank format." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "unhashable type: 'list'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0ma_dense\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;31m# Convert the matrix into the TT-format with TT-rank = 3 (the larger the TT-rank, the more exactly the tensor will be converted, but the more memory and time everything will take). For matrices, matrix rank coinsides with TT-rank.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0ma_tt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt3f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_tt_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma_dense\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_tt_rank\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;31m# a_tt stores the factorized representation of the matrix, namely it stores the matrix as a product of two smaller matrices which are called TT-cores. You can access the TT-cores directly.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma_tt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtt_cores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/projects/t3f/t3f/decompositions.py\u001b[0m in \u001b[0;36mto_tt_tensor\u001b[0;34m(tens, max_tt_rank, epsilon, name)\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mare_tt_ranks_defined\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 193\u001b[0m \u001b[0mranks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mTensorTrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtt_cores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatic_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mranks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/projects/t3f/t3f/tensor_train.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, tt_cores, shape, tt_ranks, convert_to_tensors)\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mconvert_to_tensors\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0;31m# TODO: what does this namescope do?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"TensorTrain\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtt_cores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 40\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtt_cores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"core%d\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 6008\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6009\u001b[0m \u001b[0mcache_key\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_old_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_default_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6010\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mcache_key\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mname_scope_cache\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6011\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscope_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname_scope_cache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcache_key\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6012\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscope_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: unhashable type: 'list'" + ] + } + ], + "source": [ + "# Generate a random dense matrix of size 3 x 4.\n", + "a_dense = np.random.randn(3, 4)\n", + "# Convert the matrix into the TT-format with TT-rank = 3 (the larger the TT-rank, the more exactly the tensor will be converted, but the more memory and time everything will take). For matrices, matrix rank coinsides with TT-rank.\n", + "a_tt = t3f.to_tt_tensor(a_dense, max_tt_rank=3)\n", + "# a_tt stores the factorized representation of the matrix, namely it stores the matrix as a product of two smaller matrices which are called TT-cores. You can access the TT-cores directly.\n", + "print(a_tt.tt_cores)\n", + "# To check that the convertions into the TT-format didn't change the matrix too much, let's convert it back and compare to the original.\n", + "reconstructed_matrix = t3f.full(a_tt)\n", + "print(reconstructed_matrix, a_dense)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The same idea applies to tensors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Generate a random dense tensor of size 3 x 2 x 2.\n", + "a_dense = np.random.randn(3, 2, 2)\n", + "# Convert the tensor into the TT-format with TT-rank = 3.\n", + "a_tt = t3f.to_tt_tensor(a_dense, max_tt_rank=3)\n", + "# Print TT-cores that compactly represent the tensor.\n", + "print(a_tt.tt_cores)\n", + "# To check that the convertions into the TT-format didn't change the tensor too much, let's convert it back and compare to the original.\n", + "reconstructed_tensor = t3f.full(a_tt)\n", + "print(reconstructed_tensor, a_dense)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "T3F is a library of different operations that can be applied to the tensors in the TT-format (by working directly with the compact representation, i.e. without the need to materialize the tensors themself).\n", + "Here are some basic examples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a random tensor of shape (3, 2, 2) directly in the TT-format (in contrast to generating a dense tensor and then converting it to TT).\n", + "b_tt = t3f.random_tensor((3, 2, 2), tt_rank=3)\n", + "# Compute the Frobenius norm of the tensor.\n", + "norm = t3f.frobenius_norm(b_tt)\n", + "print(norm)\n", + "# Compute the TT-representation of the sum or elementwise product of two TT-tensors.\n", + "sum = a_tt + b_tt\n", + "prod = a_tt * b_tt\n", + "twice_a = 2 * a_tt\n", + "# Most operations on TT-tensors increase the TT-rank. After applying a seqeunce of operations the TT-rank can increase by too much and we may want to reduce it. To do that there is a rounding operation, which finds the closes tensor to the given one of a smaller rank.\n", + "rounded_prod = t3f.round(prod)\n", + "CHECK RANKS!\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/quick_start.rst b/docs/quick_start.rst deleted file mode 100644 index 7764ecc6..00000000 --- a/docs/quick_start.rst +++ /dev/null @@ -1,66 +0,0 @@ -.. t3f documentation master file, created by - sphinx-quickstart on Sun Mar 12 10:06:09 2017. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Quick start -=========== - -Import the libraries - -.. code-block:: python - - import tensorflow as tf - import t3f - -Generate a random tensor and compute its norm. - -.. code-block:: python - - # Create a random tensor of shape (3, 2, 2). - a = t3f.random_tensor((3, 2, 2), tt_rank=3) - norm = t3f.frobenius_norm(a) - # Convert TT-tensor into a dense tensor for printing. - a_full = t3f.full(a) - # Run a tensorflow session to run the operations. - with tf.Session() as sess: - # Run the operations. Note that if you run these - # two operations separetly (sess.run(a_full), sess.run(norm)) - # the result will be different, since sess.run will - # generate a new random tensor a on each run because `a' is - # an operation 'generate me a random tensor'. - a_val, norm_val = sess.run([a_full, norm]) - print('The norm is %f' % norm_val) - print(a_val) - -Arithmetic operations - -.. code-block:: python - - a = t3f.random_tensor((3, 2, 2), tt_rank=3) - b_dense = tf.random_normal((3, 2, 2)) - # Use TT-SVD on b_dense. - b_tt = t3f.to_tt_tensor(b_dense, max_tt_rank=4) - sum_round = t3f.round(t3f.add(a, b_tt), max_tt_rank=2) - -Tensor operations - -.. code-block:: python - - # Inner product (sum of products of all elements). - a = t3f.random_tensor((3, 2, 2), tt_rank=3) - b = t3f.random_tensor((3, 2, 2), tt_rank=4) - inner_prod = t3f.flat_inner(a, b) - -Matrix operations - -.. code-block:: python - - A = t3f.random_matrix(((3, 2, 2), (2, 3, 3)), tt_rank=3) - b = t3f.random_matrix(((2, 3, 3), None), tt_rank=3) - # Matrix-by-vector - matvec = t3f.matmul(A, b) - - # Matrix-by-dense matrix - b_dense = tf.random_normal((18, 1)) - matvec2 = t3f.matmul(A, b_dense) From 3f09b13bccd9cc0409f5ba914739ac7228ec5131 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 17:34:46 +0000 Subject: [PATCH 136/233] link to colab and init t3f installation code --- docs/quick_start.ipynb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/quick_start.ipynb b/docs/quick_start.ipynb index b62a362b..735e5527 100644 --- a/docs/quick_start.ipynb +++ b/docs/quick_start.ipynb @@ -5,10 +5,13 @@ "metadata": {}, "source": [ "# Quick start\n", + "\n", + "#### [Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/quick_start.ipynb) this page in an interactive mode via Google Colaboratory.\n", + "\n", "In this quick starting guide we show the basics of working with t3f library. The main concept of the library is a TensorTrain object -- a compact (factorized) representation of a tensor (=multidimensional array). This is generalization of the matrix low-rank decomposition.\n", "\n", "\n", - "To begin, install t3f (InstallationInstructions_), import some libraries, and enable [eager execution mode](https://www.tensorflow.org/guide/eager) which simplifies workflow with TensorFlow" + "To begin, [install T3F](https://t3f.readthedocs.io/en/latest/installation.html), import some libraries, and enable [eager execution mode](https://www.tensorflow.org/guide/eager) which simplifies workflow with TensorFlow" ] }, { @@ -17,12 +20,16 @@ "metadata": {}, "outputs": [], "source": [ - "# TODO: check if T3F is installed and install it!\n", - "\n", "import numpy as np\n", "import tensorflow as tf\n", - "import t3f\n", - "tf.enable_eager_execution()" + "tf.enable_eager_execution()\n", + "try:\n", + " import t3f\n", + "except ImportError:\n", + " # Install T3F if it's not already installed.\n", + " !git clone https://github.com/Bihaqo/t3f.git\n", + " !cd t3f; pip install .\n", + " import t3f" ] }, { From 5994ec1ae27af756b96df251074abc1f81672bd9 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 17:37:46 +0000 Subject: [PATCH 137/233] add name argument to tensortrain and remove values arg from the name scope --- t3f/tensor_train.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/t3f/tensor_train.py b/t3f/tensor_train.py index ba59406a..79807004 100644 --- a/t3f/tensor_train.py +++ b/t3f/tensor_train.py @@ -10,7 +10,8 @@ class TensorTrain(TensorTrainBase): t3f represents a Tensor Train object as a tuple of TT-cores. """ - def __init__(self, tt_cores, shape=None, tt_ranks=None, convert_to_tensors=True): + def __init__(self, tt_cores, shape=None, tt_ranks=None, + convert_to_tensors=True, name="TensorTrain"): """Creates a `TensorTrain`. Args: @@ -25,6 +26,7 @@ def __init__(self, tt_cores, shape=None, tt_ranks=None, convert_to_tensors=True) equal to 1. If None, tries to infer the ranks from the cores. convert_to_tensors: bool, if True than convert each element of the tt_cores tuple into a tf.Tensor (e.g. to initialize from np.array) + name: The name of ops. Returns: A `TensorTrain`. @@ -35,8 +37,7 @@ def __init__(self, tt_cores, shape=None, tt_ranks=None, convert_to_tensors=True) """ tt_cores = list(tt_cores) if convert_to_tensors: - # TODO: what does this namescope do? - with tf.name_scope("TensorTrain", values=tt_cores): + with tf.name_scope(name): for i in range(len(tt_cores)): name = "core%d" % i tt_cores[i] = tf.convert_to_tensor(tt_cores[i], name=name) From 7906e870067e5431021f3dae0f3bfd7140f7d109 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 18:18:20 +0000 Subject: [PATCH 138/233] some basic info on ariphmetic and TT-matrices --- docs/quick_start.ipynb | 181 ++++++++++++++++++++++++++++++++++------- 1 file changed, 152 insertions(+), 29 deletions(-) diff --git a/docs/quick_start.ipynb b/docs/quick_start.ipynb index 735e5527..7d057a98 100644 --- a/docs/quick_start.ipynb +++ b/docs/quick_start.ipynb @@ -36,6 +36,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "Converting to and from TT-format\n", + "------------------------------------------------\n", + "\n", "Let's start with converting a dense (numpy) matrix into the TT-format, which in this case coinsides with the low-rank format." ] }, @@ -45,17 +48,36 @@ "metadata": {}, "outputs": [ { - "ename": "TypeError", - "evalue": "unhashable type: 'list'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0ma_dense\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;31m# Convert the matrix into the TT-format with TT-rank = 3 (the larger the TT-rank, the more exactly the tensor will be converted, but the more memory and time everything will take). For matrices, matrix rank coinsides with TT-rank.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0ma_tt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt3f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_tt_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma_dense\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_tt_rank\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;31m# a_tt stores the factorized representation of the matrix, namely it stores the matrix as a product of two smaller matrices which are called TT-cores. You can access the TT-cores directly.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma_tt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtt_cores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/projects/t3f/t3f/decompositions.py\u001b[0m in \u001b[0;36mto_tt_tensor\u001b[0;34m(tens, max_tt_rank, epsilon, name)\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mare_tt_ranks_defined\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 193\u001b[0m \u001b[0mranks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mTensorTrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtt_cores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatic_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mranks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/projects/t3f/t3f/tensor_train.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, tt_cores, shape, tt_ranks, convert_to_tensors)\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mconvert_to_tensors\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0;31m# TODO: what does this namescope do?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"TensorTrain\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtt_cores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 40\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtt_cores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"core%d\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 6008\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6009\u001b[0m \u001b[0mcache_key\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_old_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_default_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6010\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mcache_key\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mname_scope_cache\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6011\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscope_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname_scope_cache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcache_key\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6012\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscope_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mTypeError\u001b[0m: unhashable type: 'list'" + "name": "stdout", + "output_type": "stream", + "text": [ + "factors of the matrix: (, )\n", + "Original matrix: \n", + "[[-0.37987684 2.05245337 -0.35579983 1.2085382 ]\n", + " [ 0.88487745 -0.7914341 0.77149975 0.12811055]\n", + " [-1.02101132 1.93720209 -0.90908913 -0.14740026]]\n", + "Reconstructed matrix: \n", + "tf.Tensor(\n", + "[[-0.37987684 2.05245337 -0.35579983 1.2085382 ]\n", + " [ 0.88487745 -0.7914341 0.77149975 0.12811055]\n", + " [-1.02101132 1.93720209 -0.90908913 -0.14740026]], shape=(3, 4), dtype=float64)\n" ] } ], @@ -65,10 +87,13 @@ "# Convert the matrix into the TT-format with TT-rank = 3 (the larger the TT-rank, the more exactly the tensor will be converted, but the more memory and time everything will take). For matrices, matrix rank coinsides with TT-rank.\n", "a_tt = t3f.to_tt_tensor(a_dense, max_tt_rank=3)\n", "# a_tt stores the factorized representation of the matrix, namely it stores the matrix as a product of two smaller matrices which are called TT-cores. You can access the TT-cores directly.\n", - "print(a_tt.tt_cores)\n", + "print('factors of the matrix: ', a_tt.tt_cores)\n", "# To check that the convertions into the TT-format didn't change the matrix too much, let's convert it back and compare to the original.\n", "reconstructed_matrix = t3f.full(a_tt)\n", - "print(reconstructed_matrix, a_dense)\n" + "print('Original matrix: ')\n", + "print(a_dense)\n", + "print('Reconstructed matrix: ')\n", + "print(reconstructed_matrix)\n" ] }, { @@ -80,47 +105,145 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The difference between the original tensor and the reconsrtucted one is 0.000001\n" + ] + } + ], "source": [ "# Generate a random dense tensor of size 3 x 2 x 2.\n", - "a_dense = np.random.randn(3, 2, 2)\n", + "a_dense = np.random.randn(3, 2, 2).astype(np.float32)\n", "# Convert the tensor into the TT-format with TT-rank = 3.\n", "a_tt = t3f.to_tt_tensor(a_dense, max_tt_rank=3)\n", - "# Print TT-cores that compactly represent the tensor.\n", - "print(a_tt.tt_cores)\n", + "# The 3 TT-cores are available in \n", "# To check that the convertions into the TT-format didn't change the tensor too much, let's convert it back and compare to the original.\n", "reconstructed_tensor = t3f.full(a_tt)\n", - "print(reconstructed_tensor, a_dense)\n" + "print('The difference between the original tensor and the reconsrtucted '\n", + " 'one is %f' % np.linalg.norm(reconstructed_tensor - a_dense))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "T3F is a library of different operations that can be applied to the tensors in the TT-format (by working directly with the compact representation, i.e. without the need to materialize the tensors themself).\n", + "Ariphmetic operations\n", + "--------------------------------\n", + "\n", + "T3F is a library of different operations that can be applied to the tensors in the TT-format by working directly with the compact representation, i.e. without the need to materialize the tensors themself.\n", "Here are some basic examples" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Frobenius norm of the tensor is 0.968410\n", + "The TT-ranks of a and b are 3 and 2. The TT-rank of their elementwise product is 6. The TT-rank of their product after rounding is 3. The difference between the exact and the rounded elementwise product is 0.003162.\n" + ] + } + ], "source": [ "# Create a random tensor of shape (3, 2, 2) directly in the TT-format (in contrast to generating a dense tensor and then converting it to TT).\n", - "b_tt = t3f.random_tensor((3, 2, 2), tt_rank=3)\n", + "b_tt = t3f.random_tensor((3, 2, 2), tt_rank=2)\n", "# Compute the Frobenius norm of the tensor.\n", "norm = t3f.frobenius_norm(b_tt)\n", - "print(norm)\n", + "print('Frobenius norm of the tensor is %f' % norm)\n", "# Compute the TT-representation of the sum or elementwise product of two TT-tensors.\n", - "sum = a_tt + b_tt\n", - "prod = a_tt * b_tt\n", - "twice_a = 2 * a_tt\n", + "sum_tt = a_tt + b_tt\n", + "prod_tt = a_tt * b_tt\n", + "twice_a_tt = 2 * a_tt\n", "# Most operations on TT-tensors increase the TT-rank. After applying a seqeunce of operations the TT-rank can increase by too much and we may want to reduce it. To do that there is a rounding operation, which finds the closes tensor to the given one of a smaller rank.\n", - "rounded_prod = t3f.round(prod)\n", - "CHECK RANKS!\n" + "rounded_prod_tt = t3f.round(prod_tt, max_tt_rank=3)\n", + "a_max_tt_rank = np.max(a_tt.get_tt_ranks())\n", + "b_max_tt_rank = np.max(b_tt.get_tt_ranks())\n", + "exact_prod_max_tt_rank = np.max(prod_tt.get_tt_ranks())\n", + "rounded_prod_max_tt_rank = np.max(rounded_prod_tt.get_tt_ranks())\n", + "difference = t3f.frobenius_norm(prod_tt - rounded_prod_tt)\n", + "print('The TT-ranks of a and b are %d and %d. The TT-rank '\n", + " 'of their elementwise product is %d. The TT-rank of '\n", + " 'their product after rounding is %d. The difference '\n", + " 'between the exact and the rounded elementwise '\n", + " 'product is %f.' % (a_max_tt_rank, b_max_tt_rank,\n", + " exact_prod_max_tt_rank,\n", + " rounded_prod_max_tt_rank,\n", + " difference))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Working with TT-matrices\n", + "------------------------------------\n", + "\n", + "Recall that for 2-dimensional tensors the TT-format coincides with the matrix low-rank format. Hovewer, sometimes matrices can have full matrix rank, but some tensor structure (for example a kronecker product of matrices). In this case there is a special object called Matrix TT-format. You can think of it as a sum of kronecker products (although it's a bit more complicated than that).\n", + "\n", + "Let's say that you have a matrix of size 8 x 27. You can convert it into the matrix TT-format with 3 TT-cores of tensor shape (2, 2, 2) x (3, 3, 3) or, for example, into the matrix TT-format with 2 TT-cores of tensor shape (4, 2) x (3, 9)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Frobenius norm of the matrix is 8.127118\n" + ] + } + ], + "source": [ + "a_dense = np.random.rand(8, 27).astype(np.float32)\n", + "a_matrix_tt = t3f.to_tt_matrix(a_dense, shape=((2, 2, 2), (3, 3, 3)), max_tt_rank=4)\n", + "# Now you can work with 'a_matrix_tt' like with any other TT-object, e.g.\n", + "print('Frobenius norm of the matrix is %f' % t3f.frobenius_norm(a_matrix_tt))\n", + "2.0 * a_matrix_tt # multiplication by a number.\n", + "prod_tt = a_matrix_tt * a_matrix_tt # Elementwise product of two TT-matrices.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "But, additionally, you can also compute matrix multiplication" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Difference between multiplying matrix by vector in the TT-format and then converting the result into dense vector and multiplying dense matrix by dense vector is 0.000002.\n" + ] + } + ], + "source": [ + "vector_tt = t3f.random_matrix(((3, 3, 3), (1, 1, 1)), tt_rank=3)\n", + "matvec_tt = t3f.matmul(a_matrix_tt, vector_tt)\n", + "# Check that the result coinsides with np.matmul.\n", + "matvec_expected = np.matmul(t3f.full(a_matrix_tt), t3f.full(vector_tt))\n", + "difference = np.linalg.norm(matvec_expected - t3f.full(matvec_tt))\n", + "print('Difference between multiplying matrix by vector in '\n", + " 'the TT-format and then converting the result into '\n", + " 'dense vector and multiplying dense matrix by '\n", + " 'dense vector is %f.' % difference)" ] } ], From ea509324422ad87445a2a6a995db3cbb687b6630 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 18:23:13 +0000 Subject: [PATCH 139/233] do not make "open this in colab" a subsection --- docs/quick_start.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quick_start.ipynb b/docs/quick_start.ipynb index 7d057a98..7012ccc6 100644 --- a/docs/quick_start.ipynb +++ b/docs/quick_start.ipynb @@ -6,7 +6,7 @@ "source": [ "# Quick start\n", "\n", - "#### [Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/quick_start.ipynb) this page in an interactive mode via Google Colaboratory.\n", + "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/quick_start.ipynb) this page in an interactive mode via Google Colaboratory.**\n", "\n", "In this quick starting guide we show the basics of working with t3f library. The main concept of the library is a TensorTrain object -- a compact (factorized) representation of a tensor (=multidimensional array). This is generalization of the matrix low-rank decomposition.\n", "\n", From 519d3d1d0795a1a7b73c654d836276e908cfad58 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 18:31:27 +0000 Subject: [PATCH 140/233] citation block --- docs/index.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index f7af091b..50382698 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,4 +15,20 @@ t3f is implemented on top of TensorFlow which gives it a few nice properties: installation quick_start - api \ No newline at end of file + api + +Citation +-------- + +If you use T3F in your research work, we kindly ask you to cite the paper_ describing this library + +.. _paper: https://arxiv.org/abs/1801.01928 + +.. code-block:: console + + @article{novikov2018tensor, + title={Tensor Train decomposition on TensorFlow (T3F)}, + author={Novikov, Alexander and Izmailov, Pavel and Khrulkov, Valentin and Figurnov, Michael and Oseledets, Ivan}, + journal={arXiv preprint arXiv:1801.01928}, + year={2018} + } From 72bcfa188a299c07c9787ce4e6c076855b0db8be Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 18:35:17 +0000 Subject: [PATCH 141/233] remove parts of readme that are covered in documentation --- README.md | 85 ++++--------------------------------------------------- 1 file changed, 5 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 2bec3357..d7196d31 100644 --- a/README.md +++ b/README.md @@ -3,86 +3,11 @@ TensorFlow implementation of the Tensor Train (TT) -Toolbox. -# API -API is available via [readthedocs](https://t3f.readthedocs.io/en/latest/). +# Documentation +The documentation is available via [readthedocs](https://t3f.readthedocs.io/en/latest/index.html). -# Installation -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation of version >= 1.2, tested versions are from 1.2 to 1.11 (see [here](https://www.tensorflow.org/install/) for TF installation instructions). -We don't include it into pip requirements since the installation of TensorFlow varies depending on your setup. -Then simply run -```bash -pip install t3f -``` - -# Basic usage -Import the libraries -```python -import tensorflow as tf -import t3f -``` - -Generate a random tensor and compute its norm. -```python -# Create a random tensor of shape (3, 2, 2). -a = t3f.random_tensor((3, 2, 2), tt_rank=3) -norm = t3f.frobenius_norm(a) -# Convert TT-tensor into a dense tensor for printing. -a_full = t3f.full(a) -# Run a tensorflow session to run the operations. -with tf.Session() as sess: - # Run the operations. Note that if you run these - # two operations separetly (sess.run(a_full), sess.run(norm)) - # the result will be different, since sess.run will - # generate a new random tensor a on each run because `a' is - # an operation 'generate me a random tensor'. - a_val, norm_val = sess.run([a_full, norm]) - print('The norm is %f' % norm_val) - print(a_val) -``` - -### Arithmetic -```python -a = t3f.random_tensor((3, 2, 2), tt_rank=3) -b_dense = tf.random_normal((3, 2, 2)) -# Use TT-SVD on b_dense. -b_tt = t3f.to_tt_tensor(b_dense, max_tt_rank=4) -sum_round = t3f.round(t3f.add(a, b_tt), max_tt_rank=2) -``` - -### Tensor operations -```python -# Inner product (sum of products of all elements). -a = t3f.random_tensor((3, 2, 2), tt_rank=3) -b = t3f.random_tensor((3, 2, 2), tt_rank=4) -inner_prod = t3f.flat_inner(a, b) -``` - -### Matrix operations -```python -A = t3f.random_matrix(((3, 2, 2), (2, 3, 3)), tt_rank=3) -b = t3f.random_matrix(((2, 3, 3), None), tt_rank=3) -# Matrix-by-vector -matvec = t3f.matmul(A, b) - -# Matrix-by-dense matrix -b_dense = tf.random_normal((18, 1)) -matvec2 = t3f.matmul(A, b_dense) -``` -# More examples -For more examples (e.g. how to build neural networks or how to do Riemannian optimization) see ```examples``` folder. - -# Citing -If you use T3F in your research work, we kindly ask you to cite [the paper](https://arxiv.org/abs/1801.01928) describing this library -``` -@article{novikov2018tensor, - title={Tensor Train decomposition on TensorFlow (T3F)}, - author={Novikov, Alexander and Izmailov, Pavel and Khrulkov, Valentin and Figurnov, Michael and Oseledets, Ivan}, - journal={arXiv preprint arXiv:1801.01928}, - year={2018} -} -``` - -# Benchmarking +# Comparison with other libraries +TODO: list of libraries, pros and cons, benchmarking? Or maybe just link to the documentation? Here are the results im ms of benchmarking T3F on CPU and GPU and comparing against the [TTPY library](https://github.com/oseledets/ttpy) @@ -94,7 +19,7 @@ nosetests --logging-level=WARNING ``` # Building API documentation -The documentation is build by sphinx and hosted on readthedocs.org. To rebuild the documentation, install sphinx and compile the docs by +The documentation is build by sphinx and hosted on readthedocs.org. To locally rebuild the documentation, install sphinx and compile the docs by ```bash cd docs make html From c3bc27f86ab08d0d6d89734e6d8171f7739a9ede Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 18:36:50 +0000 Subject: [PATCH 142/233] return the citation back and merge benchmarking and other implementations --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d7196d31..b7a1fac1 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ The documentation is available via [readthedocs](https://t3f.readthedocs.io/en/l # Comparison with other libraries TODO: list of libraries, pros and cons, benchmarking? Or maybe just link to the documentation? +There are also implementations of the TT-toolbox in [plain Python](https://github.com/oseledets/ttpy) and [Matlab](https://github.com/oseledets/TT-Toolbox). Here are the results im ms of benchmarking T3F on CPU and GPU and comparing against the [TTPY library](https://github.com/oseledets/ttpy) @@ -25,5 +26,13 @@ cd docs make html ``` -# Other implementations -There are also implementations of the TT-toolbox in [plain Python](https://github.com/oseledets/ttpy) and [Matlab](https://github.com/oseledets/TT-Toolbox). +# Citing +If you use T3F in your research work, we kindly ask you to cite [the paper](https://arxiv.org/abs/1801.01928) describing this library +``` +@article{novikov2018tensor, + title={Tensor Train decomposition on TensorFlow (T3F)}, + author={Novikov, Alexander and Izmailov, Pavel and Khrulkov, Valentin and Figurnov, Michael and Oseledets, Ivan}, + journal={arXiv preprint arXiv:1801.01928}, + year={2018} +} +``` \ No newline at end of file From 8f86c2be8a2e33977012f2fb3b08b3cb97e9ab1a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 18:42:22 +0000 Subject: [PATCH 143/233] create tutorials section of the docs --- docs/index.rst | 8 ++++++++ .../tutorials/riemannian.ipynb | 0 .../tutorials/tensor_completion.ipynb | 0 .../tensor-nets.ipynb => docs/tutorials/tensor_nets.ipynb | 0 4 files changed, 8 insertions(+) rename examples/tensor-riemannian.ipynb => docs/tutorials/riemannian.ipynb (100%) rename examples/tensor-completion.ipynb => docs/tutorials/tensor_completion.ipynb (100%) rename examples/tensor-nets.ipynb => docs/tutorials/tensor_nets.ipynb (100%) diff --git a/docs/index.rst b/docs/index.rst index 50382698..106c9f20 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,14 @@ t3f is implemented on top of TensorFlow which gives it a few nice properties: quick_start api +.. toctree:: + :maxdepth: 1 + :caption: Tutorials + + tutorials/tensor_completion + tutorials/tensor_nets + tutorials/riemannian + Citation -------- diff --git a/examples/tensor-riemannian.ipynb b/docs/tutorials/riemannian.ipynb similarity index 100% rename from examples/tensor-riemannian.ipynb rename to docs/tutorials/riemannian.ipynb diff --git a/examples/tensor-completion.ipynb b/docs/tutorials/tensor_completion.ipynb similarity index 100% rename from examples/tensor-completion.ipynb rename to docs/tutorials/tensor_completion.ipynb diff --git a/examples/tensor-nets.ipynb b/docs/tutorials/tensor_nets.ipynb similarity index 100% rename from examples/tensor-nets.ipynb rename to docs/tutorials/tensor_nets.ipynb From d84ef24c2a798550348c5131728029c455634121 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 19:13:11 +0000 Subject: [PATCH 144/233] remove grand titles so that the list of tutorials is correct --- docs/tutorials/tensor_completion.ipynb | 16 +++++++++------- docs/tutorials/tensor_nets.ipynb | 20 ++++++++------------ 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index 18c54c5a..59fae6bd 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -40,7 +40,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Generating problem instance,\n", + "#### Generating problem instance,\n", "Lets generate a random matrix $A$, noise, and mask $P$." ] }, @@ -72,7 +72,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Initialize the variable and compute the loss" + "#### Initialize the variable and compute the loss" ] }, { @@ -97,7 +97,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# SGD optimization\n", + "SGD optimization\n", + "------------------------\n", "The simplest way to solve the optimization problem is Stochastic Gradient Descent: let TensorFlow differentiate the loss w.r.t. the factors (cores) of the TensorTrain decomposition of the estimated tensor and minimize the loss with your favourite SGD variation." ] }, @@ -187,7 +188,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Speeding it up,\n", + "Speeding it up\n", + "---------------------\n", "The simple solution we have so far assumes that loss is computed by materializing the full estimated tensor and then zeroing out unobserved elements. If the tensors are really large and the fraction of observerd values is small (e.g. less than 1%), it may be much more efficient to directly work only with the observed elements." ] }, @@ -398,9 +400,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Environment (conda_tf)", + "display_name": "Python 3", "language": "python", - "name": "conda_tf" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -412,7 +414,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.6.5" } }, "nbformat": 4, diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index c09da0bf..38f9c1a0 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -23,7 +23,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Tensor Nets\n", + "# Tensor Nets\n", "\n", "In this notebook we provide an example of how to build a simple Tensor Net (see https://arxiv.org/abs/1509.06569).\n", "\n", @@ -243,7 +243,8 @@ "collapsed": true }, "source": [ - "# Compression of Dense layers" + "Compression of Dense layers\n", + "------------------------------------------" ] }, { @@ -503,13 +504,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Finetuning the model. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ + "Finetuning the model \n", + "-------------------------------\n", "We can now finetune this tensor network." ] }, @@ -563,9 +559,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Environment (conda_tf)", + "display_name": "Python 3", "language": "python", - "name": "conda_tf" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -577,7 +573,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.6.5" } }, "nbformat": 4, From e5eb324e208df7bfb00c8d81deb1fdd19714d9e9 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 19:13:36 +0000 Subject: [PATCH 145/233] ignore jypiter checkpoints --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 5047eb36..973f8ca6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -78,7 +78,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' From 2a7bb32ad4c22e752fdbee852a5d66f0d8a6e1b7 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 19:17:57 +0000 Subject: [PATCH 146/233] add colab links and t3f installation code --- docs/tutorials/riemannian.ipynb | 49 +++++++++++++++----------- docs/tutorials/tensor_completion.ipynb | 43 +++++++++++++--------- docs/tutorials/tensor_nets.ipynb | 39 ++++++++++++-------- 3 files changed, 78 insertions(+), 53 deletions(-) diff --git a/docs/tutorials/riemannian.ipynb b/docs/tutorials/riemannian.ipynb index 99a45649..8963d17e 100644 --- a/docs/tutorials/riemannian.ipynb +++ b/docs/tutorials/riemannian.ipynb @@ -1,27 +1,12 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import t3f\n", - "import tensorflow as tf\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "tf.set_random_seed(0)\n", - "np.random.seed(0)\n", - "%matplotlib inline" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Riemannian optimization" + "# Riemannian optimization\n", + "\n", + "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/riemannian.ipynb) this page in an interactive mode via Google Colaboratory.**" ] }, { @@ -48,6 +33,28 @@ "We can implement this in `t3f` using the `t3f.riemannian` module. As a retraction it is convenient to use the rounding method (`t3f.round`)." ] }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "tf.set_random_seed(0)\n", + "np.random.seed(0)\n", + "%matplotlib inline\n", + "\n", + "try:\n", + " import t3f\n", + "except ImportError:\n", + " # Install T3F if it's not already installed.\n", + " !git clone https://github.com/Bihaqo/t3f.git\n", + " !cd t3f; pip install .\n", + " import t3f" + ] + }, { "cell_type": "code", "execution_count": 2, @@ -261,9 +268,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Environment (conda_tf)", + "display_name": "Python 3", "language": "python", - "name": "conda_tf" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -275,7 +282,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.6.5" } }, "nbformat": 4, diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index 59fae6bd..cf66739f 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -1,28 +1,13 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import tensorflow as tf\n", - "import t3f\n", - "tf.set_random_seed(0)\n", - "np.random.seed(0)\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Tensor completion\n", "\n", + "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_completion.ipynb) this page in an interactive mode via Google Colaboratory.**\n", + "\n", "In this example we will see how can we do tensor completion with t3f, i.e. observe a fraction of values in a tensor and recover the rest by assuming that the original tensor has low TT-rank.\n", "Mathematically it means that we have a binary mask $P$ and a ground truth tensor $A$, but we observe only a noisy and sparsified version of $A$: $P \\odot (\\hat{A})$, where $\\odot$ is the elementwise product (applying the binary mask) and $\\hat{A} = A + \\text{noise}$. In this case our task reduces to the following optimization problem:\n", "\\begin{equation*}\n", @@ -36,6 +21,30 @@ "\n" ] }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import tensorflow as tf\n", + "tf.set_random_seed(0)\n", + "np.random.seed(0)\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "try:\n", + " import t3f\n", + "except ImportError:\n", + " # Install T3F if it's not already installed.\n", + " !git clone https://github.com/Bihaqo/t3f.git\n", + " !cd t3f; pip install .\n", + " import t3f" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index 38f9c1a0..48bdf239 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -1,5 +1,21 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tensor Nets\n", + "\n", + "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb) this page in an interactive mode via Google Colaboratory.**\n", + "\n", + "In this notebook we provide an example of how to build a simple Tensor Net (see https://arxiv.org/abs/1509.06569).\n", + "\n", + "The main ingredient is the so-called TT-Matrix, a generalization of the Kronecker product matrices, i.e. matrices of the form \n", + "$$A = A_1 \\otimes A_2 \\cdots \\otimes A_n$$\n", + "\n", + "In `t3f` TT-Matrices are represented using the `TensorTrain` class." + ] + }, { "cell_type": "code", "execution_count": 2, @@ -8,7 +24,6 @@ }, "outputs": [], "source": [ - "import t3f\n", "import numpy as np\n", "import tensorflow as tf\n", "import keras.backend as K\n", @@ -16,21 +31,15 @@ "tf.set_random_seed(0)\n", "np.random.seed(0)\n", "sess = tf.InteractiveSession()\n", - "K.set_session(sess)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Tensor Nets\n", - "\n", - "In this notebook we provide an example of how to build a simple Tensor Net (see https://arxiv.org/abs/1509.06569).\n", + "K.set_session(sess)\n", "\n", - "The main ingredient is the so-called TT-Matrix, a generalization of the Kronecker product matrices, i.e. matrices of the form \n", - "$$A = A_1 \\otimes A_2 \\cdots \\otimes A_n$$\n", - "\n", - "In `t3f` TT-Matrices are represented using the `TensorTrain` class." + "try:\n", + " import t3f\n", + "except ImportError:\n", + " # Install T3F if it's not already installed.\n", + " !git clone https://github.com/Bihaqo/t3f.git\n", + " !cd t3f; pip install .\n", + " import t3f" ] }, { From 63c354fa2b97678a249d24637beb9acd6ffdd491 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 20:44:20 +0000 Subject: [PATCH 147/233] init faq --- docs/TT.png | Bin 0 -> 22142 bytes docs/faq.rst | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 3 files changed, 53 insertions(+) create mode 100644 docs/TT.png create mode 100644 docs/faq.rst diff --git a/docs/TT.png b/docs/TT.png new file mode 100644 index 0000000000000000000000000000000000000000..8898a5290bd275fcf5d40ea0751085afbb3c4d13 GIT binary patch literal 22142 zcmb6AbDShivjzhz#KRNSAk~#dFtW51p`huIK&W!v3e^0k%l({dkmCTk!s1ZWuEq*I4LKwM z=sM^M4s?FB)P`ypAYMPz*WUzdudR-Vp08F@Cb#)QS zK>4e3__0vWhNLBB4dOgOz90e6%|X7iiX|N4;^Twl{Ce=3Ho$S|i*lV7!N+%~X!3yb+W$yDf zc5L5>R91}miAk)!TI~WsgzbnM$7QFFsr{#|xO7s2%swPm%OXZH(Kr_ECY}|RKJpY8 zhjJo1W*EcoVGLI-ieAv2@l*)m$!m!A!XNe%05AZW{R+qq(kcHqlG)Jf=x;DB%KP94 zbFwa?Z?aKyK`cgl*E50wvc^H&leN;iN0ghrs9y%7Rts(e1&KovLkOrwe@P`=3s0fl zppp=)1~f}0;N#Li{*X^NCgiXrMH>=l#}#^G)xkm|t?T*bB$Dr8(1_RQOs@`$RZhf0 zFMJR|el0A4oR8Ps?#GV}`D+Zdf2_rzruEHmud~pDW5&_T8kT{AU9@p>SwO+yti(h-wVtbI3mjcq7M!2DK&v$M%D? z-b+)67CsIG@EQpqKcSKkAVoeuONe>oAf0smvM`aujiJYT7l!POLKoSZQ5RPCXz#Ha-WF_AI#MNOx^I+tH#_Tai*xE(TLTQDL zhFXH|2wEco;f@uJ)tcDGg!R>*1dr!u_=Q0>MGrrCM%d6JLu_TXj`$exM(md1Zv}?C z4sE&{@!Gwfk^A5kvdd(9prGFPTvI& zcw{aae=0J<1+hP)0^GT6Ig3^sv{XZEyJL#c(k2{}tueg~ycW7koteH4J^%ubFsLoCw9gpJ+s!I?f*oiHR9eBOrkk zUHp43eyNa;NEC5#OoTX_Bd$l-9cgu}9A${2f{QZdcM*yl+Y$zxsKMO1X}E8Mbb?aT ziRS2<5f?(ZGfGD$cbuyco&2GUepH5V=D z1txAc@2+r06pI6{dOEQ?_hirQ^_M4@cATw1wq3KU@+Tp0q)*8lWM?QZK0OgrS2`Lwi9Bp(;aT`H4v$6(^!0U7ClNlTo;y->#sn{7uC4DrMdZZxIiE5rF~a0lnbLTK+=O{ghFeL}HcJ zvJT4x%c>KYg^n^zmF)7^dFchvg=R}~ORjm@a-Je1XVO-_Od&nNEeSrUOC^f3^wQ8W zv(n`9OSMa7!g={Q`&senluCts2krc1e@Pow;t$)0St_SG|-SoAm znjh2teAh_ho?xDUm^e+#u20eQ*Id!Gs!yxGu==x_v>M-FW??tOTh7C%`=kBPee$XH zI&^SrM!<#E#nAcS>aucbNw3&f z@e{EKVlFwgufj0P00HL+D+ntLz7f|M2M#w4dk_}`mlbQ1Ef1>&yAZdM#gYx@+ag<* zG51vS=2KL7KzO=D(FjER?bl10329WB@E?du-gllqYnQAVo>;4w3)UJ3^O|NXNgkHQ@1U=O}O=nfc)k_yl_e>9Mgj<7J zGhF++mW=v~qCi@mW#=uZ+I7;E!_z!~|?v(XZN7fWYn#`WeI>+_zAPc zv9JM?b3>lnhO3F2jlDK8U#-*BP1GC=Jo!>hPJJBSxq(R7Z@zW9p@r$yp zm=-8hVb(uE6>=4<@-M{FMGnP2SyS0LJRK+YikTjmZVe}gW{u4HFCu?n9%438U)Z*L zw6k~e2yhr!3hMN}_WSk!?B6Vambi_lr?8^9jZaq+r-G(xQw%8rm(}1av@5%vpDkr8 zN|j0HIp1|YKjxT=o|4b%$#P&%j7dvrWxHqL`l&NUAPFn8mcTlEeQ3SizFpgV+WgMV zp!?-gqgBmQ1^m(}~Any=*>8ILr$a;yjDI15g+D@nfX)2wxOF+B^CRsy9X& zgTx zY7Wx1Dch>)YJ7NnF&ik%KKHG9@h~=d8J$$2FK?{)T7^;3`Fqf8A`52WXWq{tD~d`R z?ICTo#*6b8>{A9+P8Hh?{uQFlX)Sbxh4vZuKPQpd?65HL00hmU-6eYHxMfope^cgS@6AF5@k4cAHc3K_UD^{J6F{sb)h6CbZ-`bA9SY zjwelznvG8SP7>*a1DVbYoK*4YZPCH_R(99c953nTc5PblnpRO(czP}Yj(L0i!y|d| zLN*>YV8zU`Jhs_0K9A0`$5v)vvyZuPeI9ySa-52%UbFZ-dQJsTMWQ2y=v#CwThAO@ ze)qNfIl7$eB0P6Kwr=0`ENmI;C@=Ifd<-YZ&QekXtIBG_@I1e{A3oh++qGM7<+wM! zq`Rm*Y;v;gYE^PKd%DQ;3i2{}RJl;&o%WD_MU3$McGr4G;EQ!t^GD$oxGD?IYzzY%8K2vk$D$x9p?kxbS(5)c5zt36?a@nHRbD z_Z!av+)B2#EzS-b!k+}5SM-k?SFg2*nuHJD&adttO1s5oeRaI?ylOvwKh55Je>?4n zU5}0BZj7PhfzhwYJw67sZ^8rUu(R-60XdW-7rKf9#*ZRYpr6oV7iNn76v-R3fL%-w zB#(f^0|F8ymad4i6`1 z#+=yzR&Pw)c*Bu)o_*4T!CiEHUXkC3KEV}2Y^Fh+8*zY{0<^ujh7$mQLH_p(BB4lj z1pt6#S}3bKtIJ4p8rj*<8yMRen$Wx3*#Au|0Nn1Jz@m+bvjLI2jkT>4r#lbHe_C(? z%YSb(kP!W+iL(_CiMotDk&vCE3DI}@Z}f~Lyf8#WMBI+XrkskxqW_WuuXsqzot^DD z85rE$-00m{=r-|3j(qzm!aD%>S42|J3|T$<6Rrf&Zh>Kcn@Z zyTIY%h2duSpP}c4S-^8}1zH2oLRd~2cn1Az8sJA2_@(^!3@l^qR#5LF0MAtt!UD?f zAZMAd&S)a7Lwq*@00CXlo6bR60sQuWypo=s+%XZn_Rd6i@JF05M6CJkLG+7_l@9Jb z%3lFs&0c^nQ=1}HeitLH*7Qi)3XEZ>o8aU#HAqC@Um1zx3+!AdZc8Ml^35!Jr4P0a z_X^5e%UWL^Ri@sGUrwu6C9FxUi2$Jg?%h!JfsEbk01%{qc6$^*M*FyK-+wOwAmdp2 zaDD)&fA(f*{lV@DP_KW9ksiT`;C6kW0RLOhkJYT9j|k2W_`C0B0fGe95rNt7zwFsS zpupH*&;0*s&%ci9#~AiX3;_JgZ~H62VB6H6=U?GrUZ6-24t(H%|1bmcTL+3;r>%kf zxAP8ifL@}DPypcH{`sW?g8b9JJp3b^pXdUbAFKZz2>|%7Za{JNb=?I2D$~yc0u|Uy z2(;F}{A_{ZY%!L>{^_iLGX=l^=}`y(_?KU|450V_yYl|xASH!2aX;#-_ZTzTy<#n` z!e7ibC0~B$m)ISW?H8Cm_z3!z5=bZwA^cLNn6{M%*Dudu6pPFH(oF~%+QHj1LLp)ZTEF2qf!-&CZ%J2RJt1zDDatt{8Wc=%ZX-@P<~ z+1o7v`q}qYpMfU9(OFhNKV}gVKkiJsTxHfYzK0yic&BF7@{c6Z3@1?H;tuTzZA~&| z+{w8g!_=2vze9iJFmURhv=~WJ%JM{HNP25H^)(`O1g-%BMb@jp80ax1m}vV9z*z|Z=+!(0IhJ670&Z+-PnD&PTprh ztS8|>Fn?(Ff&lN#Q{(`0bzcy#-@6%D&d=TW0^xo@0e)e5WdNW6C%Vhukf#JYzfE|6#(4nM`}8J;d%NvX$OgAGpV<4B zR&)f)gRW|u|9k0$w&IxodGdR`v6q`B;}Spqw*_GX7tmcB6)#7I%I7~~chULDrH6jb zt*={f->7I%9~9aA2DPsr$_yS}j5jEh)o)PilA~-1d9vV8LzK)Sd8}{;25$G8=Lw2d zNS%%^zq+ zujwfJnSPeocc!AMItUuG&&C5fpZeCWTCrN!7)zDHfH**J!^*mWc(9v8Sal;KEC9m` zp`ag>at7)| z9K|(&lxS8pN^yj8ZBqTn!8TOce3qthApI+Wy5I)71p|hmwGcubW?kw)UuiL8*NASC zp`M#jn2wMC-pBcrUVERdQ5~3dQBJKo zGiN@@$ql&``B+`~LakIe{8=DeuOB3~tYksuGL5dM23Cyj>i%`6VIAv0En^DuE+?K6 zm5az~#CLNYJL3mt@nH1H*9rPcI|Sy&mzYrGz6)a=429DI-f9vr#0p%kcUgwsZ# zHP!SlW0P2k7L5cdnXX`BIL$Z&J__bW$A>Ze0~{+o_SB$9z4neX#oPfhY^RLJ3IQ0_ zGjVq{l*|T^+l_`W9_ERait!9ivne9Dyt#)Xl>%sYVT3vCbX%BvB~|f=j5=5}abbR} z5*9cXHj+lm6Q$qeTF5_ML_`c+Q5JN-gVx4~6Ez|V4Ii3T)~%~E7Oqcesbi{T94^LV zCDXI#fuTT@$#pl8U&UdrK9w6aXY4oQ7RXeI2tw(NeC}G4Um|g!o!G$F5G06L9I0I! za=tEGOEc$qEK{C5D_JH-M`JsOoIyu}4&YyYa7>WEFltP&quRv_I$tyus3teGKzfgCAlbQT z;M{7RJtfbSEq)u}i(B{Q03oasC1n9;H^k`w8q)oI(^vnwt-z_HB;=AH_~QjJcbph| zz9~J4h;X7uj9zoN(MB`_LLb=((YG?ls=UK0D;If=UQuB{bo3;>Ame0DuUM&zBky zc|u$_=JwPIilGQhZbl6{MhkOGG`a-}E#(PP-;VC@{{07fKE#%c9o@C-tIM=(zO@YnRqJP)# z;8gT_Gm`8ufEhD+gwRsc#TXS9<=j&jOl4v<0b&XhZ88BMh{!odE-7ZAkee-aG7FEZ z&ix3SlbA#8Y=i_?+Y1F^2SRps@PDzuwI~o6Vy&DmTefaF{q5g}FB7xYbT?20kHxb;L z6<4RlQN(-;2M0%;tkKhQtFvR;l|iRfIFr)_WwmHs{ctj~6fD9K`#gQTzO_};osG0d z1z};#@LEYy^5^u&*RNj{*)UGnFd8E;8CnG~BF^lR>2)YMI5dpo73MI41m;sf@x#qP z<#_JVHr)S23L>AaHAVY;Je#F`vxu8^U8>L!QYw~3Up+wNd*j#8(70Q7nu4}rLy7!x zJHckmoO(3Lb&0n6w%TBIykeG(zG^xXMPg}gox@}_7-rK{64q$DL0vx~{{5jF3~?IY zZJoH@qkN(4*Frcd1&j@wCQcw@7=~01p@WS6VugmWt|i8({#=nv{b#q2=PjEZjE&ky z+)ATO`ZTKv^6f_FrTJ>Th2UDGuu&}|C6B74Qqk~^-GI%XOI$hMtrs^3_}Zi-Iz7=5 ziIHzVUYv8!Cz)%nEit%sgI%Zde`~M|Jl;nzbjpKc2Khspp9KJ*!pH#ze>b9sEXPI& zgKaY8H{WqA@F^H_0U|@7UH@(*PF@07=7%r_Ihoj&etz~q^Zwm>?Pv-GpuosBzhtW;#JM%{Gv0Z|HI|u$rR9Zu5oH$M=(IPac?7{G{}O*< zAt@7*tNxPz$LsI=L!o7QWxbr+g+q(0oF46StE!d|=`I!r4>hjeUXQk%u6OFNgn?#y zKB;A=CxRE7O#@rAHl((HPCo0-z#A@?V_WssBE}F|6+V5@A$^@HTOPTK4ATC2 zsnXd1QwklZQm#6qyCeE)O4A%q9xlCN3Qprc^SO1W}sgpz(NHjQN~&+f&_iwSPN)fuF#W zeBAV6Sh^@xDlU;D+!TnQna(IFEzOlmW>C1)?P%$<>$+v_csVK8&dO-sPtpns^|W*Dg&=g%Te_DMw$}yRlzem8+%uN^Wl=N6Cb^)uM z_;Pzl70cn(C?&pq6DTv!w1uj)$n(v37^zC~ajJ3f;kIC;?U_YwqXdf+<_EXp)6e|ac#WQfNNUqYE5u9n?%!wxsNC%7d- z)LkFlJ%Nx~L3+BK-W(ggpYMC;B_*yX9y+1*_1`Y0<@nwUDzc|Lyxi!0KLVcbL_ct~ zvo!NMbF*-8akTpH_#AeHVHqAHKiguHXf+%cI<4!W0;Q~$rc+K|6qmAi7U;S3vjU;Z z!7<|bVmnf|TWr)@g9unAAI4|v^@h$8clVE4Vnrm{qk|7FW_iB#%nWZ*j~p##YKaNy z7U!CqA&*S@s9C|PVhvj*8vmM6qQDU3m>#5;TG?ac}S@sAFpAu}^GrOgcPR+D8q5USy>q;_8_ zd=AH;JvG=4Ge|fHFq@THa{ct$u-MnMw6xJg9{X*;hVz_^cGerrA!#%(3TkhT>$_)K zcDfYLa`l0HV=JL=f$&#ql#ZZfZj7jv{#68*HpeH%hDXmxm&tg!XcyB znrd5b*zIkB`15_#TC>%;l%;&((+U}aRL?sW1w1r5DTVUIbHxh@qSDqQ0}0$P0}uof zhU>E-8QnFoOE{KxOj zL!bxw-RK!d(3v4T?a+@e99t=+34|`^3|2S{B zEHU|lH^pmhm~dt4kFkF4KCJ23jB?fT@jE&)Cz~~`QBg64$0F%qII~G`?bueTI~iZBE-bb()H63y{vk;x)A~m??e<1WyNe`t zde^(;z{uXeL!suW5Yyu7{urcf$-GPYJWi=ZM-fHsA?4m{^y&t3<#u1T^*xKWDx+Gs zuMkJ_tUnZt_vW3qH7l4y_Q9QHe?3)_Q$tm@b?r%h!tz|wMXdz3JbiCQEu4&V{0%$A zGg$5;_E{GEk5`^6K_zfJG{b@MfFcBk2nhsq2PV+^0cQ7uBFk8hL!N>yrhlek*%zD`@^04;MwfYlarI<+xKpG+_w7m(<`XB{rKezG{VpMGxErra|vx4O{KX*bu1Y!^CgebZzWD-3>p-nVw^MVFgx*N1TkSNLD;=SYEJMHB+qo}=3<~!V zFM)pTTLr6T`OTkbK3U)j==$*$I=$~^y9XV~h|G_Bbk3*wx!=TahU?Gv zT(EXD(d9svF?C~BG-`@%?Ht;=^Du$zj=bJ}!R$VJv106P-uDY?cYE=2jYtsjskfFl zux~gh+5<=^ND##QLW#pjF~L2a7|bqFOOPtfX7QLKE5D7znkVe<-7eNk+gt11uMO$> z)`L(uQ{G#ng5jf4dg~OecqLb z9JeZa)P06dez0V^=#?fEZ|Fc>Ar>l&X?+B_o0huzr>3sL3m(tbn{xF{H*4#zn=(u_ z4D_`200BvOcv^lkSc`LHUQOF0ELfI|od)ZhJqg6=gQ?G5&gd@ua5{%`eA8#Bw&*z) zX37y#&x59MkcZnT-uzGI)TaY=y)_S!sstR|64;2MBkVHGDzr#mi09|$yQWIte0qzE zsoudsQBGxiH&TCiWabemXX63EzjUt z(pvu212>Mv?jgL*2eh{G-?co;79DjewvHLRy*tNGSHdVu9YRlppJ95A?7v$=tQ`dx&V)W^l#B zz4DDPj?7B$C_h#`I61pm1Y@(~8~gVfb6aSF(-N!WsP`iU$&(bxk#|V@g|qP)1{Xl~ z5IR&$^V@#%{51hDp1sY!g=>{;!d`>1wO^5T=;?A?lLjrC^-GubXW_h4#+Zs6%eKW15vf5r)GC z`&F9>b_h)Za%Bdeh}JAlnR6v2o#aFPt_l`Ngo1T_!OKOqZyW?)86kzECw#6+3SAwz zCPzYb)0hI2D%E!sMjxxvZVf6YM5*w(6a@|Q%6I+W$?r@QkY#*{?Nhd0$2}Or@8{O2 z6l|^chGVN(_cP7!n3|oBQSR}J@@+LVwDlHHYO^VsLai>XDxKIVgX!YWTQbews4G-Y z@OI6T%(N;SK0kpkFg`yq8k_b200#fRZ!XStV6_ueRzCxCb|uMmnHiMJF_LNuZ4AgG zTN4|<&sUjJSvO-#V(Vi@YF5_J)+ko+PH$kq9It`;>P>%XN|LIfOA`FSLSCRwNNF0A z1ILm7t8>fg;H8u0vU}YRn{$Y8(qqc2>ho6juIHU#$68-G$4@t zI1A7Db3P>AZr=zlav*)@>Lir$d0~k^@K&?oC7uHL)`DEu_kyF1^E(cF=j#R6+W}kG zZ>}MfJ5Qd6^R>{UBlDHp#&A9F>n{id>|rKjNfNRXEuT|Inu%l*T#3mM6iMrWGnl&O z5|`#yeIIhXJ*3fmwh~Tg9K{R7eixN~#6n~m2xapr80b_IZB9F&?P{%BaR*1+cslw1 zq@v=ogz}~K$NR`o1g!IMx^0iJ-dCGm?k~?WN#DefbK7upg+dL(Wb_d6*!@aMC?%Ns z?mJ~XJJMOaU(cJGGqS{@1dWVH={xQe+g$PX5dKtns!g?unzOuO~xzxi?Cd)e{E2M9gWJKoguXdWypI^kl@%%#{|_LtA{(IN^M7 zgeYprX!>h!9#!gMFygYEM#5=-+xZ7ih)gy;Ra|P?&$gq%%maERb_YCr_9!M0&ZgGFbQeibU%nym6Qy`SUOmxO`3C6;Du@oZN zRgAm--r?6S1IKymW_XwOJ&NmMFqIUsda%Hz`T>_|nB1{7Wblef=Wt2atFmC|7h3BlrB_yTpU^d_ER*Up`8LTfmWMrm8Zn2#sg?^vW=fi zS7%|TuFw5KTzk`0b~NJXD49hR!sP5dhjGE(oVBkGevys`IdY&ulA#yhC*I}dYVh5- zv{YJtnElJQ$^6HO|T^0fz={kAG#vB%~Lm5@^Rm(#&E3c2FB#4Y&DmP8dU7g7U>tn5rNVp5l147)9p zNFjfrtvd|iLnwCZ3Qop>suEH_z?9bPk_BnWa~UD_gI zKKV?ZppEQT61xU2FO>~Vn+uFwbHY*YJn_Rx2B@m~OZBJ4;V$1v2NEUWJve|$jB5dv zp#KVa+4)KjOqW5wHhu*4_hV|?`+Gy;6*wAh1F$IKy+|P{(#*1dU}!a3GsOg^)>}6^ zA1lIAr7-q9Poc@e?NJQ;Wiu%bh=(Jipujzt@E%V(TCCP%*aCqp&A?&@6ou~b583|jB;vN?Mp+-&-`JDpFOaN{F)Z1taMPA4ST){z-}Lor}z6cbPt2Pww4x~ zM2Cezfl*<7AN3|5eH5TyP%^9u>R+(*j9LAevA^obdVVIyw`0811|{V=y+R_p&-F4N zpR(Pb)}WM_%A83m$a+}L@Z0G2fFR;G!*r!q({Af&70VGmqeM@CII0(|k=Lw};nm!4 zqo<(7=6C|n;YQ20KXvsN_~LmUTKIvN=Er}Q$R035pV}@(d6AXPK(+F)4D*fbuhOd> zMof~Wl0Z8vDhF&e>7~40j&;jTI3Y+S2zyt@h3y9{_Xy>BN7PQS(b^L<=L zFN=&}m)8G!sz-gVsZ2lx(-QXNd*EbZ>`Ks>`^ipjUobZt@y8hzr|<%X1U?(dg3*VX zkXkdmG{P;XUZoqfPIh@_BZbR>m>L-}k(Dsk(7`Wv(3Bh7o=I2V;rC>~2ht+A+`!^I zGnCdr>gr;?_n(QBuH3lVGWm9DlwZo=sHFgj@AYI0jLKV$xC(zT1nH`#lmolo?G1_e zj5fbQSj*f!tl159eZJeR+xNjgUG*W7b3)#9)X3c@U{RWP+#^*QNQV1sRI8Wm0aFnT za0to7mC5NBOQa4zP%tpMOs=&;0j^4}=*X6lIMZJ>o`o(jy&OU~N=ay;bHy%LtkRyh zjjF2p>oPF7_+TK?@T-m7b1pB92??UWw0QL; z!>l*bJFGvhLiyMy&$fRR_isZ{3G#N_>1yNhtO>y8?bVT~7&{pT3Sc77#nklKsVz{ztF1Dxf5L*DB29 zV%~X|yG#wQth~G@0m=(@t;vq<2)Dn#e|t`j@0_mf%tIE=*x;CHrma862+^?5G{0cz zYa~UT<#%z`a5!La8K_8*6rS~L8?D{6Rn<+kM#X7~B!tWMTZD_2 zLxNvq!Sq_e!p~XaN<8Ec+RxL!RRXXrL}8Y1K$Mt`ZwJwqQvxXC37S#3|5IC_^rxG; zhZ_^ltTDr$O+pX`7WNwW7~ndKj2~vc@Ar3W6jVH~e}#CdVs!?O3|9H#Cw%*xER8}{ zUcS0#7HW*szpx>y)dng((U4u1&bmKIL<#O1!38;M_zg}f%NOLfcR~--W;lZj;gtG3 zTtMqyRFir-Pi5IpZW!EAqc{vJ2aG&5{krS$9l=U!L_h`Yo}5CY?9dBy(hJrk)S|r0=K!p z!DUjbzq`WR9hjhv-3W+@h3*I(AaHE=^ZRsW$ehaz0qcgv1w1@BzRIM5-_{0{@A;vE zJ!>4p;)9vW^hyU#jKN&Qfcv=jtS@nCLRBn*RwqG1 z`epouze0&fRoq#`u{+%V&6^{A6_p7X7*oj>XKKYoM1qBj`72_~twxVZY=B*ly~bc- zj0dn#F*Xg(rmFOwER^fS;n^GNi%vi5J%)j3;P=NZfM_wx7Ze#ixZz%ba^*BUyWYrc z%il|G`6c$u{qCFYVn-BGHbNV+4n13gc)Z##gF->uceIjU~4Yi zxkIzIhwC5aul_tO-*ME%spy#@A{qVj7qxOlCMl=Tqzq?T?#l zKzZIiIstj`kEg$5KXMJh#P0o6(?kDZ=lhyoaSuWYNuyf9Xx=TC#jVjv$_k8O%YfZo z1G#=bjw-`NC}~BaV)JH)KcemTi&_w06fcJK_201=KMBX9LA;*FGze4PvEJS2^;3ID`}{DSmEtAv9%#?}Or0AIaDtuXGb zf9L`;rkhS6*VF{ebNmi_p(WUOj*yOnz~@ka>2V~1(5#hYWDr!RDQ32-7M^D=qbrBg z7`ZjRRlkYaZ?fh+~AVvfGU}?t|)Rr5TGP(}2Gb=!>5o@P%b)*W3kR@!VY0a1dfVRlf+$G+2zN zya<0}Miy>(Jtg~XvsQ4Byp}Bt#3|hRb{iPEGKf*Tb9s-RYZ!{sq~N%P8Ysg3&AzWC ze`K_Suo751kCN9;p@0$Idz(i$9#Ux=ZgYlQNWe*-AGL8#=K}vAbl(eGADBTBJRxYN zitb82#S|NCS^?dR?^a*VXqSLsV}oPal0Kez6e?XFz_p;xg%n-LK(XCGtA*^|0`ci~ zfA>m6f=%8xtOrSZ*Y?SSKPX@x1?`32lzMHiz)9I0l_>{wP-6Y!nk4cOOAyQM*6Y-` zp(TB5*TzN7@r^n;%motq>WboIsKe(?*}}AR90^2mA54bC@GI2ts@Q16!xauj%!X_+ ziiXd?7%+ody291?)LK&-~v6H!%lxU2)*xqI&hWk zPhaUcVc!ht(A6B(_S(wmGvd@?$FXc;2(#I+(7L=LGfL>h>EiCB*TulM^vwN!56_?3xu`l9Vs)qTWv$z2GdSQ^^-{D+r1+E(Vo$_BYjdWl`{0lqwWyz*RK!5!lN*h zm%_DKK?d`Zn?st+U_u5fPd5xnpk-380g$)3aedk2*svghSUgYE$c0FRQL^ZYCYt$i z7K1h1QeA|3>d}Q`@Q8OMVtBQFJB7_UjiQyuc$%n~mOdj_z!98Vm&(p$k3p3+Rg6TL z!}F(DCeyS@t)n=z{rs_MCm)gTrXEM)xj!*nyqN)HNzb=~Y-0{gj|`%)urTBufH4iG zhX?m#BaK{EP_IS!0yexSHdM`MwowfdpDD%ZGkd1t=P{)QSd%0Xi}-3XFvYH_2@qrg zL%tXwuPyL{gmgp!GrbwhB4!a*OrERP8+I|4#Eog54O~2na5U79hL*bq17~N-fmE@C zsjV$wSo_G}!&f*_ZKvq28Cmewx%#8|suz~YBhJYSNgU?Uo{`KvEHizFpw{a>fkp}%d5#o` zSL0+Qzw=(~*GIK=H8ml#;T37Bri`@LZ22FageO3eH3c6j2#9X^5@rJ8J{bfBGYgXE z9#lv;Eud8*_X?)AQNr`8XLCpn#$oSu34sRRgC0*-%R8d)1Ty{#M+Z#yiz^f$NGA?3zv+#IOM|&a-`LF{OKmG~0-*pw&BKTm9a((?J z;%_I4rQ``F-g!_;jR;WnT|QIO>j=1`37n~4OPSu@-XGZoe`e<9<{pq`f9|7zWoMIY z_5hdjs*nOd6nXbhz@BNM8OVqKFk_KHEX55-Z%Z=J5XR;G$y(BFs0)e{n_*MrV7iL* z*zR}=CItdGGfcl~Nw{W}0ACGqAS!D|1lFE;^h-H`w+HxDqXfnqCo-mKk&7RsNWqcW zt0Oe`cY_9#k=Td(gCqKMBX2{ohXW`eWI^|daFemFWD@YSaqI`QW2NAJQ96eE=a+{U zJAuC-gUE&VR7zchq^%$PJl~i&k!z$@4B-E%1X3{l6=vLPKZe*p?yj!jSJ87V0_Hlq zSVFwWzV=kA z!<(Oju%fwFcD^P9FW=m$nb;TCfBotg7M{A;kVwwqGrIG}(pP~p4&?gy!}P1uzTG(E zQ|SreLbF{MxaP=z#4Gc&Oj=?RC+hPFOf)uhu$#a9b-!;yhJ9H++M`K4(^^9+-0*PH z5Q_>2g4sku!90JWc`yo_MiOLnbnJ@!+3+YDIGm@HPQXZR>Z6gDpNlh)7>ewWjxw{O zXk+?`Esr_B-=^@QfiFu0{%1(GXMt@!Y$bN5$fot(jI5Cd2pQHtiru}X7}EUoPlYxb z;nkws*0}6zpLl+0bgnPt@f>{a6eimvJJnhLsD4b@N{xS+l+~77-Qr3u-{(chLJ1^9 zBplS5;tqVx=uT`AW{u%VPo^vX(>tO#mQlRO8+FLS#`XQhsn`Vy&vad7QQHqW={G8U zZ%mH96G|ts&C66-d91jkM+XFne_uHGDv;msmsj8_7a3LOcPkp+qIZ%PQhT6wqkqF~ z>Sy9~DKa5SaOS*|Nfpchw6&NdA1uC$Iw~uyR_WU4yt8)r4|fBra0jHH&huSe1jrww zZsusfUUu4tk5`~HNBO`1*rj4({~&$zHgDeiF-^{_C&{RL2?W?%kAVy#OZ@%%t^QaA z$qoAN?a)4~gZaSx4f%9(jQ>Xu5Vai1cSWY>C1E!$sv74Ol3;(ZGT#dSO)Gjk^s0=y zUEuI^IDnuE3VY}nxa>4&DxW!X%9%40qCZbr`ZP@lu~IlGV2;M2K2jp^xN1I9DZJXKWUhP#9@L)1Ep-%NbR!@5Vq7&0ig;#lfCm7hj4#VW4vOgaDO9f zhyiS!HHCWT6J%BNK#k@ znL|6WeWM=T&ILV9i#SGhs(~IN%Y7Hxyg^f<3z2|3EU~`hX1-L3?sz=&nz^+pq*B*L z;#M^1mFWuNXFEbQ@w9wB#2hAz3l9c~!uk^-W${yPNLCfLh{tEQTUa*_v}90(s!;1m zBeApUPlF(bFZaSYByiU~SJ@_O`&yXdAx1Iu%ht&F*>Rxvapbxbzy$<(&>zu|P9JY( z&XsW&RNGWc`lcx$KWffzI94%wK{GR1%XczAL=_5ll?r6ES$H~nn}wu5Dg-rTj5YZsM10x zNQdSNmx55# zl&vr)srkyBJx^9PCAAe$^UBuRKA*F6$}fAn4yVj$P8I^$qVz_|Dk{?tK|&r|8vHbs zBy!a!5=Q{ZJKdj?&b{a|>$>h;9Yjz(;ut3d_jjb2!br4-_`xBf5y(I}zY$sv1yP~q zluin(k8_?#Q{x2F)s&qd%V*VLU;4HnYPn&dzt&o=#d6R6`0eu<*$gXURW-C*c_JVM zgO&6PYj1yl=cK$UiGOT-pV=m%hroB|i#PCRT)GQP`u*F&o9S4uSRPb&hUPv;0zq95 zty|QDoB@WS(r_W;8RQ&4ihTI8d`aN*M;2;0@a(MmI)B1&pMX~#rX2Ql6YsstpP)Q< zvu$lNE25!6?%TjB+$fZ?j~KkIAG7Q|%eMlSukM#pJXiLia4he=o9!-j+vPSHvVUB7 zgu0~4LEQ!0^B&WILu!rHIlVq!4xy|4KUJJ{RFvJ?#s!8NM`;-8l#*tMQ97g<2I&sz zkQPv20EHn&8l;hw5~L9jaA-tIN|6*8q`UbBzjNMqo#)T{S!+MJ*S>z&b+2t|HDq@J zS?fDU3}aLMQWUoNqhV>FS-_)@Bh^q#q+#AYuDItmWAM*}l~^K9f?u-nlM6}!2URN$ zKJK{2c5@qxH?Q~J>;!kDT3h!7Mn=uvy2kq4N%p^DP74fjRRokUXZWNT)&R0kh!0WR z3Uv()KYZ9Pr@h0ThMu7JdKkN|0#gIBz}_p{=rxElTGO9iW^czyW9Dc*myN*npwnJ} z#^X6QxvIAA0-B2cS|>k=u#uVH3~lJB{T+Ba)beK-YkPNSzOMzJof#VkOH>o^iT;x# z{lbNdCa={EQ=07`4c~HO#9uTXqRukQ$$?nuHoB6US(N06A6VYp5wYd(<)km|kq}o+PPf7I>}d zp$I;HLMgZ%$1p8GArd|t z;QKwFLp7tzZ_~eXn^wy3b4PdG7H6(#B+Ez|6;|5C>CLgwK9`ly7T%!UNd#ztHaPpS zqH!}2AG=Uys1tj7bs-T{RFy9rx~${6n&W7gfZex8)l8}!<%opwI$`M*@?AU_gSvBc z?ltqDsl>3V=|Sa340?6qcrtz_wA&Jdyscp_7*j9D|mhPhRWSR5tC z8|a$13#Y*2eTJ=7gKew%mj~}C!8s(woZf$doHO>Ir#`^)-^pA6l~^>FlX!!Ad91#&CDcEX0+&Mx zJR4nRYKi>>+Dtxf?g%9$T`nPk$3>i`F}LEZz$N!7^>dEMREM9H zSoZCuYsxn-gav%W1z$Zn+5>_>wNG}x8(1t@T%XY} zdUb|&!h7#~UY)M0N0L3fo4+(`_62*2t)@Ufo%13s6(UhL)AoAe8uy0NCy((t_Es>_63>W<0J@@ zMr8X~kYs7Sxjug`Qk2LPb}$<7sIoD<6vP_;uEo2f=IA-5O+JJoEHU!5mi&Xvv8jdJ zMqR*PJI0wtmq`zelw-pAL-M_}FHgre}X3n@{PObLE>-{ zyx_Kq`#C5r2*NI$mgLWAByh7?pUGgr;|wyWb{2(475(t5TIDddWiel1g;}5oJvLz# zebt$vw+P}Mnw59+vj{pe`Y`c>CRwNqPzXm(kJPjpzf@I4=9#5@dI;WN_g}0(kO6Jw-^zG*!yh%Og3B^E+L!yxF&Z{@ATM^ zz15vem^F5(ztlA~bGJC`?5D(+pSL_h)&6Kp(Hz#b>Z?F3Y~Ew!jY~&G@24{4Y9ng{?i#*;EGnh7Boxnq{)!R>qByfkZ0-j}Lp+twD81#}_@^WOBD7$fZRECv6 zC0nxJbHjp63JMh^x(DfpcvQXtjkfST6m}YJ6*K4Tb| zJBArBSS#4)g@0AFrJm926Z{qqjW|N9S_dw2>Ev@O*NQkBxsm#thyVSM2Q0W1^3)W_ zZVn)DjOM+JI_QkFB%ifsSw5{J)*GPhz=2Yz$k$m6ctjr$z=>ZB+3Az-bL~{C|-?nPsn*cVH11My;G=$zJ(3* zDpoJp#SFLdP%F?EP1gYaY#*tj7>N4aPjMpRl&Z<1_u}t4=v8zX+OHH6j<##x{K(QV zvM;Sl^>9G(@*eq#Ie5~K?&lSb3l_!j7;|pc2jQP)p+hEo#|HmwAJDzke{fRvD=*iT z99Sa~9V%(}Y_q?%)gqPE zZWoG&#^Befm_T*kF^#evv3hpoW2btE1#>PH@GXI-)BC&zy&?!UJfQthU#Q=!rA@Wy z&?*h#Q{XBu^}`J4>1J4+F6I&<>2_6$Cn0pVDc%Pa!knpJI5_i7P+zeNt+_oAoPRQ- zHa6yF9+qk{_noM&uu0y@I+HXtji(?HY2=#dzcXYSNt@6A+GO84eISO`o;Cryv5uib z6C>-`fbIT?+q?baxaXi)_QGflKJNv_^=-*fVAp$%VrF=x=v)4e&*frc-6EgEggx9U zx$uD6i$o`)xCfoKcGgX?gZxeADx$4mVW+8#URvX_80%lPKZWJ=OgL;glxdlk)eJNS zn(-_2{~YBg+pq=J`)boVR|v;&Yr42$iqW#kN)o(4fY`%W{Fsel^|Zm$f|@aYTi&Sw zNgb{-cgpnjvT)HaLgAt&_%ht^g#0bH<5lId??lF{nm;%;fwq}xZUMVhG=;q0-9_&Z zlHPlaDQFTZYbs`);UfjMog92~*{W6js$3j__R}@kY zcH2AR*{69~I?T1@>6bqeB&m17G0B$(NlKn>TP9LvqZA}ovCcc_P=XTJ^=MA>T>Bq7 zm}G>f_#P3t)k!$0@udV+8@1IZH%)x_HUVE!Fy;QvL!({hj)urvmB!niac~QUtf8gY zaX$5xiY)H$vtb%x8Bcwbf0yS_Xn_jAdu*r>CKlPnVa22Gs*EJAsbr#%`y#@x6q7#a z+)Iw%*dv)QF;N=>+4OShj--0E!rV`sDc7A9qjXrcq!1o^nDev>^E5sK|+ z#^~4@eqO9SY*NO^n(ZmZjs(mH@6L_DjZkPOJyaboTch;WUQBh|rv(MKSa4j#08bTy|oXE4! z{sbtdRTSGgNj|A1(4}!tmbA&CR0XE+xbqs$h%Y54t;ZImHRO@2a6IkjQ(HrcDFprk z#4vCdBi}3duaFN~7|55h@lAcS45fHEY~^PM1l~nFc{EE@tYrn?&^7$b)rskJ@^qk7 zx*JRG=;f<{Ns}Q;5o@X!cqNJvL)+U2P(073OKvj#M?WXT|5I) z-}j%L+oQ()m(dQAx?}jnxkT{ z!HF`(aoJ3R&NDl!ZBkj5va*qgJc!h#`+3{Ol!Dk^%^k5lqh&pT>715z%y=XTQrPf` z?W9wclO5*-KtmB6Yt!mL-@M}K@@r4}DA`pBNef#pg)JNm)ZONwW9p(K)FN~D^7nkU zJ|(1q3dYZ%MqZE`7XRX(XbEKFH*dso?24T#NJ`<3B9Xni2%zNr%N^j5aPw`Jy)jay zAlhuU#R~StEDCjL98*?;OQFZ!7P_BPQU&2f%JgOmr|+D5U0zy+h*y)&Y(@thw_^;` z+P~dMl(y)Q?nSy;hn%1Mws9q}k+3b+v~meS`wo26YKG~Vuwgd?FuIDmEevWV zGotoNxLQvx-aPOnsiP~GUIc!fc}UA*;%KV&OzCuZ%p9uQCibKJaV}-4aRJZkLpyxK z0BLsP?oqD_)1F-W7bixjkTT`nwRNOQld zDj*q_`E+!xn9$f%IWKpQjRCKQ|1ta9B;VVb1WGVTFZeD!@uuGWhI(M0no}op-^3hY zdO)I>^jEx>KG2i>3h9RtI^T}tqFP;+(7i`XLH4Eit@VKZ2rabU44PhDeq zeM5{#3^^wOs(Qg5yNt$$Ow*5CR1Nu5L8|>zJ53WMZT!?@Va9(H+!^VTDl8;yqbp!U zz%!9Fit+=&E9#cmJbo^A0YZ>KqmP2 zHcR>^u{(E4=w@dRDClG6wIKQ$S=vwmM_8RSN|hxK%u%KSjgyGO_52dFA+y6U^J&O( zt#>!g<7-`H+VCgo_{PahaOB9m2dX_+PWFwIB#D)8f`__kp*&*y4U>3`R)v08=9=YL z=#HF-aWGw^5hk{&R!i`$i}yw!%jPaStr~55vpCC7{d;3`1d_dHKa{AUex@y?$x$HD zWNF;)ny^rURdx?+!CroIX?j!$4i3ZSH^$B*`!@x_PG_8VmIaB*GON_$QWhWoQV1DT z6~E8s`$dau5$-|NU**I3H9?r6&fhUj(&-sCF^I4To+!v~73kIQ;B-8Q%dsjt)&7}& z;Q_J!lt|5RIc?%zi^Y9$*s{-G*1!=b9rhc6569b_AjRaN!58cryGxsIXTj0}i}^QX zl27sF)4ybbkj0V-LN=?R?;kQ@Qo@o6Qx^Ca-#-`v0^E=ZV7*22pVA4}4VkQ=cijFZ z6XqK-VTP@`{!1p0Z^+~^=WD;{f5`-KLna8Qd*VMO=_qVT8Y 40). Hierarchical Tucker is very similar to TT-decomposition. + +Where can I read more about this *Tensor Train* format? +------------------------------------------------------- +Look at the `paper`_ that proposed it. You can also check out my (Alexander Novikov's) slides_, from slide 3 to 14. + +.. _slides: http://www.slideshare.net/AlexanderNovikov8/tensor-train-decomposition-in-machine-learning + +.. _paper: https://www.researchgate.net/publication/220412263_Tensor-Train_Decomposition + +By the way, train means like actual train, with wheels. The name comes from the pictures like the one below that illustrate the Tensor Train format and naturally look like a train (at least they say so). + +.. image:: TT.png + + diff --git a/docs/index.rst b/docs/index.rst index 106c9f20..e6c5be86 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,7 @@ t3f is implemented on top of TensorFlow which gives it a few nice properties: installation quick_start + faq api .. toctree:: From 71002054e6e3adb24c8d7fb12a53ffd7838a5417 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 18 Nov 2018 20:44:56 +0000 Subject: [PATCH 148/233] remove autogenerated comments --- docs/installation.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 5229a79a..8c02d608 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,7 +1,3 @@ -.. t3f documentation master file, created by - sphinx-quickstart on Sun Mar 12 10:06:09 2017. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. .. _InstallationInstructions: Installation From ae89d0bf8fa6c6e33a020ce0e9df8928057d8085 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 19 Nov 2018 20:10:40 +0000 Subject: [PATCH 149/233] nits/typos/ets --- docs/quick_start.ipynb | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/docs/quick_start.ipynb b/docs/quick_start.ipynb index 7012ccc6..0eeceb70 100644 --- a/docs/quick_start.ipynb +++ b/docs/quick_start.ipynb @@ -6,12 +6,12 @@ "source": [ "# Quick start\n", "\n", - "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/quick_start.ipynb) this page in an interactive mode via Google Colaboratory.**\n", + "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/quick_start.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", "\n", "In this quick starting guide we show the basics of working with t3f library. The main concept of the library is a TensorTrain object -- a compact (factorized) representation of a tensor (=multidimensional array). This is generalization of the matrix low-rank decomposition.\n", "\n", "\n", - "To begin, [install T3F](https://t3f.readthedocs.io/en/latest/installation.html), import some libraries, and enable [eager execution mode](https://www.tensorflow.org/guide/eager) which simplifies workflow with TensorFlow" + "To begin, let's import some libraries and enable [eager execution mode](https://www.tensorflow.org/guide/eager) which simplifies workflow with TensorFlow" ] }, { @@ -39,7 +39,7 @@ "Converting to and from TT-format\n", "------------------------------------------------\n", "\n", - "Let's start with converting a dense (numpy) matrix into the TT-format, which in this case coinsides with the low-rank format." + "Let's start with converting a dense (numpy) matrix into the TT-format, which in this case coincides with the low-rank format." ] }, { @@ -84,11 +84,16 @@ "source": [ "# Generate a random dense matrix of size 3 x 4.\n", "a_dense = np.random.randn(3, 4)\n", - "# Convert the matrix into the TT-format with TT-rank = 3 (the larger the TT-rank, the more exactly the tensor will be converted, but the more memory and time everything will take). For matrices, matrix rank coinsides with TT-rank.\n", + "# Convert the matrix into the TT-format with TT-rank = 3 (the larger the TT-rank,\n", + "# the more exactly the tensor will be converted, but the more memory and time\n", + "# everything will take). For matrices, matrix rank coinsides with TT-rank.\n", "a_tt = t3f.to_tt_tensor(a_dense, max_tt_rank=3)\n", - "# a_tt stores the factorized representation of the matrix, namely it stores the matrix as a product of two smaller matrices which are called TT-cores. You can access the TT-cores directly.\n", + "# a_tt stores the factorized representation of the matrix, namely it stores the matrix\n", + "# as a product of two smaller matrices which are called TT-cores. You can\n", + "# access the TT-cores directly.\n", "print('factors of the matrix: ', a_tt.tt_cores)\n", - "# To check that the convertions into the TT-format didn't change the matrix too much, let's convert it back and compare to the original.\n", + "# To check that the convertions into the TT-format didn't change the matrix too much,\n", + "# let's convert it back and compare to the original.\n", "reconstructed_matrix = t3f.full(a_tt)\n", "print('Original matrix: ')\n", "print(a_dense)\n", @@ -121,8 +126,9 @@ "a_dense = np.random.randn(3, 2, 2).astype(np.float32)\n", "# Convert the tensor into the TT-format with TT-rank = 3.\n", "a_tt = t3f.to_tt_tensor(a_dense, max_tt_rank=3)\n", - "# The 3 TT-cores are available in \n", - "# To check that the convertions into the TT-format didn't change the tensor too much, let's convert it back and compare to the original.\n", + "# The 3 TT-cores are available in a_tt.tt_cores.\n", + "# To check that the convertions into the TT-format didn't change the tensor too much,\n", + "# let's convert it back and compare to the original.\n", "reconstructed_tensor = t3f.full(a_tt)\n", "print('The difference between the original tensor and the reconsrtucted '\n", " 'one is %f' % np.linalg.norm(reconstructed_tensor - a_dense))\n" @@ -132,7 +138,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Ariphmetic operations\n", + "Arithmetic operations\n", "--------------------------------\n", "\n", "T3F is a library of different operations that can be applied to the tensors in the TT-format by working directly with the compact representation, i.e. without the need to materialize the tensors themself.\n", @@ -154,7 +160,8 @@ } ], "source": [ - "# Create a random tensor of shape (3, 2, 2) directly in the TT-format (in contrast to generating a dense tensor and then converting it to TT).\n", + "# Create a random tensor of shape (3, 2, 2) directly in the TT-format\n", + "# (in contrast to generating a dense tensor and then converting it to TT).\n", "b_tt = t3f.random_tensor((3, 2, 2), tt_rank=2)\n", "# Compute the Frobenius norm of the tensor.\n", "norm = t3f.frobenius_norm(b_tt)\n", @@ -163,7 +170,10 @@ "sum_tt = a_tt + b_tt\n", "prod_tt = a_tt * b_tt\n", "twice_a_tt = 2 * a_tt\n", - "# Most operations on TT-tensors increase the TT-rank. After applying a seqeunce of operations the TT-rank can increase by too much and we may want to reduce it. To do that there is a rounding operation, which finds the closes tensor to the given one of a smaller rank.\n", + "# Most operations on TT-tensors increase the TT-rank. After applying a sequence of\n", + "# operations the TT-rank can increase by too much and we may want to reduce it.\n", + "# To do that there is a rounding operation, which finds the tensor that is of\n", + "# a smaller rank but is as close to the original one as possible.\n", "rounded_prod_tt = t3f.round(prod_tt, max_tt_rank=3)\n", "a_max_tt_rank = np.max(a_tt.get_tt_ranks())\n", "b_max_tt_rank = np.max(b_tt.get_tt_ranks())\n", @@ -187,9 +197,9 @@ "Working with TT-matrices\n", "------------------------------------\n", "\n", - "Recall that for 2-dimensional tensors the TT-format coincides with the matrix low-rank format. Hovewer, sometimes matrices can have full matrix rank, but some tensor structure (for example a kronecker product of matrices). In this case there is a special object called Matrix TT-format. You can think of it as a sum of kronecker products (although it's a bit more complicated than that).\n", + "Recall that for 2-dimensional tensors the TT-format coincides with the matrix low-rank format. However, sometimes matrices can have full matrix rank, but some tensor structure (for example a kronecker product of matrices). In this case there is a special object called Matrix TT-format. You can think of it as a sum of kronecker products (although it's a bit more complicated than that).\n", "\n", - "Let's say that you have a matrix of size 8 x 27. You can convert it into the matrix TT-format with 3 TT-cores of tensor shape (2, 2, 2) x (3, 3, 3) or, for example, into the matrix TT-format with 2 TT-cores of tensor shape (4, 2) x (3, 9)." + "Let's say that you have a matrix of size 8 x 27. You can convert it into the matrix TT-format of tensor shape (2, 2, 2) x (3, 3, 3) (in which case the matrix will be represented with 3 TT-cores) or, for example, into the matrix TT-format of tensor shape (4, 2) x (3, 9) (in which case the matrix will be represented with 2 TT-cores)." ] }, { @@ -210,7 +220,7 @@ "a_matrix_tt = t3f.to_tt_matrix(a_dense, shape=((2, 2, 2), (3, 3, 3)), max_tt_rank=4)\n", "# Now you can work with 'a_matrix_tt' like with any other TT-object, e.g.\n", "print('Frobenius norm of the matrix is %f' % t3f.frobenius_norm(a_matrix_tt))\n", - "2.0 * a_matrix_tt # multiplication by a number.\n", + "twice_a_matrix_tt = 2.0 * a_matrix_tt # multiplication by a number.\n", "prod_tt = a_matrix_tt * a_matrix_tt # Elementwise product of two TT-matrices.\n" ] }, @@ -218,7 +228,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "But, additionally, you can also compute matrix multiplication" + "But, additionally, you can also compute matrix multiplication between TT-matrices" ] }, { From b7e59fe5adc924dedf00859b25f9ff8d55f03581 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 19 Nov 2018 20:11:57 +0000 Subject: [PATCH 150/233] fix showing space adter "open" link --- docs/tutorials/riemannian.ipynb | 2 +- docs/tutorials/tensor_completion.ipynb | 2 +- docs/tutorials/tensor_nets.ipynb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/riemannian.ipynb b/docs/tutorials/riemannian.ipynb index 8963d17e..28814e52 100644 --- a/docs/tutorials/riemannian.ipynb +++ b/docs/tutorials/riemannian.ipynb @@ -6,7 +6,7 @@ "source": [ "# Riemannian optimization\n", "\n", - "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/riemannian.ipynb) this page in an interactive mode via Google Colaboratory.**" + "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/riemannian.ipynb)** **this page in an interactive mode via Google Colaboratory.**" ] }, { diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index cf66739f..8e530c57 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -6,7 +6,7 @@ "source": [ "# Tensor completion\n", "\n", - "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_completion.ipynb) this page in an interactive mode via Google Colaboratory.**\n", + "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_completion.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", "\n", "In this example we will see how can we do tensor completion with t3f, i.e. observe a fraction of values in a tensor and recover the rest by assuming that the original tensor has low TT-rank.\n", "Mathematically it means that we have a binary mask $P$ and a ground truth tensor $A$, but we observe only a noisy and sparsified version of $A$: $P \\odot (\\hat{A})$, where $\\odot$ is the elementwise product (applying the binary mask) and $\\hat{A} = A + \\text{noise}$. In this case our task reduces to the following optimization problem:\n", diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index 48bdf239..c143fb27 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -6,7 +6,7 @@ "source": [ "# Tensor Nets\n", "\n", - "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb) this page in an interactive mode via Google Colaboratory.**\n", + "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", "\n", "In this notebook we provide an example of how to build a simple Tensor Net (see https://arxiv.org/abs/1509.06569).\n", "\n", From 3b28a697c3f68d4222f8fc4059f586ee07550f3f Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 19 Nov 2018 20:25:25 +0000 Subject: [PATCH 151/233] Better subtitle structure --- docs/tutorials/tensor_completion.ipynb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index 8e530c57..1c93eff4 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -49,7 +49,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Generating problem instance,\n", + "**Generating problem instance**\n", + "\n", "Lets generate a random matrix $A$, noise, and mask $P$." ] }, @@ -81,7 +82,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Initialize the variable and compute the loss" + "**Initialize the variable and compute the loss**" ] }, { @@ -106,8 +107,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "SGD optimization\n", - "------------------------\n", + "## SGD optimization\n", "The simplest way to solve the optimization problem is Stochastic Gradient Descent: let TensorFlow differentiate the loss w.r.t. the factors (cores) of the TensorTrain decomposition of the estimated tensor and minimize the loss with your favourite SGD variation." ] }, @@ -197,8 +197,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Speeding it up\n", - "---------------------\n", + "## Speeding it up\n", "The simple solution we have so far assumes that loss is computed by materializing the full estimated tensor and then zeroing out unobserved elements. If the tensors are really large and the fraction of observerd values is small (e.g. less than 1%), it may be much more efficient to directly work only with the observed elements." ] }, From 4983d1de0980e9ddc176b5ddffde2c186b7edc5c Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 19 Nov 2018 20:31:44 +0000 Subject: [PATCH 152/233] start with NNs --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index e6c5be86..d32664bc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,8 +22,8 @@ t3f is implemented on top of TensorFlow which gives it a few nice properties: :maxdepth: 1 :caption: Tutorials - tutorials/tensor_completion tutorials/tensor_nets + tutorials/tensor_completion tutorials/riemannian Citation From cc61254bf5bf53141d4aa19f578f7669ecf0a9ef Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 19 Nov 2018 20:31:58 +0000 Subject: [PATCH 153/233] more informative names --- docs/tutorials/tensor_completion.ipynb | 2 +- docs/tutorials/tensor_nets.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index 1c93eff4..4a1667c0 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Tensor completion\n", + "# Tensor completion (example of minimizing a loss w.r.t. TT-tensor)\n", "\n", "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_completion.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", "\n", diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index c143fb27..6d90a940 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Tensor Nets\n", + "# Tensor Nets (compressing neural networks)\n", "\n", "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", "\n", From e5a8a60d8a2c9d39b8c6d32ecb848afb095f0ce7 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 1 Dec 2018 12:05:28 +0000 Subject: [PATCH 154/233] use bencmark_session and ron_op_benchmark to correctly profile tf code --- examples/profile/profile.py | 196 ++++++++++++++++++------------------ 1 file changed, 97 insertions(+), 99 deletions(-) diff --git a/examples/profile/profile.py b/examples/profile/profile.py index 32bf17e9..c6e73eca 100644 --- a/examples/profile/profile.py +++ b/examples/profile/profile.py @@ -25,109 +25,107 @@ vecs100 = t3f.cast(vecs100, tf.float64) one_vec100 = t3f.get_variable('one_vec100', initializer=vecs100[0]) vecs100 = t3f.get_variable('vecs100', initializer=vecs100) -sess = tf.Session() +sess = tf.Session(config=tf.test.benchmark_config()) sess.run(tf.global_variables_initializer()) print(device_lib.list_local_devices()) logs = {} matvec_op = t3f.matmul(one_matrix, one_vec).op -# Warmup. -timeit.timeit(lambda: sess.run(matvec_op), number=10) -matvec_time = timeit.timeit(lambda: sess.run(matvec_op), number=1000) / 1000 +benchmark = tf.test.Benchmark() +logs['matvec'] = benchmark.run_op_benchmark(sess, matvec_op) print('Multiplying %s by %s takes %f seconds.' % (one_matrix, one_vec, - matvec_time)) -logs['matvec_time'] = matvec_time - -batch_matvec_op = t3f.matmul(one_matrix, vecs).op -batch_matvec_time = timeit.timeit(lambda: sess.run(batch_matvec_op), - number=100) / 100 -print('Multiplying %s by %s takes %f seconds.' % (one_matrix, vecs, - batch_matvec_time)) -logs['batch_matvec_time'] = batch_matvec_time - -matmul_op = t3f.matmul(one_matrix, one_matrix).op -matmul_time = timeit.timeit(lambda: sess.run(matmul_op), - number=1000) / 1000 -print('Multiplying %s by itself takes %f seconds.' % (one_matrix, matmul_time)) -logs['matmul_time'] = matmul_time - -batch_matmul_op = t3f.matmul(one_matrix, matrices).op -batch_matmul_time = timeit.timeit(lambda: sess.run(batch_matmul_op), - number=100) / 100 -print('Multiplying %s by %s takes %f seconds.' % (one_matrix, matrices, - batch_matmul_time)) -logs['batch_matmul_time'] = batch_matmul_time - -norm_op = t3f.frobenius_norm(one_matrix, differentiable=True).op -norm_time = timeit.timeit(lambda: sess.run(norm_op), - number=1000) / 1000 -print('Computing the norm of %s takes %f seconds.' % (one_matrix, norm_time)) -logs['norm_time'] = norm_time - -batch_norm_op = t3f.frobenius_norm(matrices, differentiable=True).op -batch_norm_time = timeit.timeit(lambda: sess.run(batch_norm_op), - number=1000) / 1000 -print('Computing the norm of %s takes %f seconds.' % (matrices, batch_norm_time)) -logs['batch_norm_time'] = batch_norm_time - -flatinner_op = t3f.flat_inner(one_vec, one_vec).op -flatinner_time = timeit.timeit(lambda: sess.run(flatinner_op), - number=1000) / 1000 -print('Computing the dot product between %s and itself takes %f seconds.' % - (one_vec, flatinner_time)) -logs['flatinner_time'] = flatinner_time - -gram_op = t3f.gram_matrix(vecs).op -gram_time = timeit.timeit(lambda: sess.run(gram_op), - number=100) / 100 -print('Computing the gram matrix of %s takes %f seconds.' % (vecs, gram_time)) -logs['batch_gram_time'] = gram_time - -tens = tf.cast(tf.random_normal((10, 10, 10, 10)), tf.float64) -tt_svd_op = t3f.to_tt_tensor(tens, max_tt_rank=10).op -tt_svd_time = timeit.timeit(lambda: sess.run(tt_svd_op), - number=1000) / 1000 -print('TT-SVD for tensor of shape %s takes %f seconds.' % (tens.get_shape(), - tt_svd_time)) -logs['tt_svd_time'] = tt_svd_time - -round_op = t3f.round(one_vec100, max_tt_rank=10).op -round_time = timeit.timeit(lambda: sess.run(round_op), - number=1000) / 1000 -print('Rounding %s takes %f seconds.' % (one_vec100, round_time)) -logs['round_time'] = round_time - -batch_round_op = t3f.round(vecs100, max_tt_rank=10).op -batch_round_time = timeit.timeit(lambda: sess.run(batch_round_op), - number=100) / 100 -print('Rounding %s takes %f seconds.' % (vecs100, batch_round_time)) -logs['batch_round_time'] = batch_round_time - -project_op = t3f.project(one_vec, one_vec).op -project_time = timeit.timeit(lambda: sess.run(project_op), - number=1000) / 1000 -print('Projecting %s on %s takes %f seconds.' % (one_vec, one_vec, project_time)) -logs['project_time'] = project_time - -batch_project_op = t3f.project(vecs, one_vec).op -batch_project_time = timeit.timeit(lambda: sess.run(batch_project_op), - number=10) / 10 -print('Projecting %s on %s takes %f seconds.' % (vecs, one_vec, - batch_project_time)) -logs['batch_project_time'] = batch_project_time - -project100_op = t3f.project(one_vec100, one_vec).op -project100_time = timeit.timeit(lambda: sess.run(project100_op), - number=100) / 100 -print('Projecting %s on %s takes %f seconds.' % (one_vec100, one_vec, project100_time)) -logs['project_rank100_time'] = project100_time - -batch_project100_op = t3f.project(vecs100, one_vec).op -batch_project100_time = timeit.timeit(lambda: sess.run(batch_project100_op), - number=100) / 100 -print('Projecting %s on %s takes %f seconds.' % (vecs100, one_vec, batch_project100_time)) -logs['batch_project_rank100_time'] = batch_project100_time - -if args.file_path is not None: - pickle.dump(logs, open(args.file_path, 'wb')) + logs['matvec']['wall_time'])) + +# batch_matvec_op = t3f.matmul(one_matrix, vecs).op +# batch_matvec_time = timeit.timeit(lambda: sess.run(batch_matvec_op), +# number=100) / 100 +# print('Multiplying %s by %s takes %f seconds.' % (one_matrix, vecs, +# batch_matvec_time)) +# logs['batch_matvec_time'] = batch_matvec_time + +# matmul_op = t3f.matmul(one_matrix, one_matrix).op +# matmul_time = timeit.timeit(lambda: sess.run(matmul_op), +# number=1000) / 1000 +# print('Multiplying %s by itself takes %f seconds.' % (one_matrix, matmul_time)) +# logs['matmul_time'] = matmul_time + +# batch_matmul_op = t3f.matmul(one_matrix, matrices).op +# batch_matmul_time = timeit.timeit(lambda: sess.run(batch_matmul_op), +# number=100) / 100 +# print('Multiplying %s by %s takes %f seconds.' % (one_matrix, matrices, +# batch_matmul_time)) +# logs['batch_matmul_time'] = batch_matmul_time + +# norm_op = t3f.frobenius_norm(one_matrix, differentiable=True).op +# norm_time = timeit.timeit(lambda: sess.run(norm_op), +# number=1000) / 1000 +# print('Computing the norm of %s takes %f seconds.' % (one_matrix, norm_time)) +# logs['norm_time'] = norm_time + +# batch_norm_op = t3f.frobenius_norm(matrices, differentiable=True).op +# batch_norm_time = timeit.timeit(lambda: sess.run(batch_norm_op), +# number=1000) / 1000 +# print('Computing the norm of %s takes %f seconds.' % (matrices, batch_norm_time)) +# logs['batch_norm_time'] = batch_norm_time + +# flatinner_op = t3f.flat_inner(one_vec, one_vec).op +# flatinner_time = timeit.timeit(lambda: sess.run(flatinner_op), +# number=1000) / 1000 +# print('Computing the dot product between %s and itself takes %f seconds.' % +# (one_vec, flatinner_time)) +# logs['flatinner_time'] = flatinner_time + +# gram_op = t3f.gram_matrix(vecs).op +# gram_time = timeit.timeit(lambda: sess.run(gram_op), +# number=100) / 100 +# print('Computing the gram matrix of %s takes %f seconds.' % (vecs, gram_time)) +# logs['batch_gram_time'] = gram_time + +# tens = tf.cast(tf.random_normal((10, 10, 10, 10)), tf.float64) +# tt_svd_op = t3f.to_tt_tensor(tens, max_tt_rank=10).op +# tt_svd_time = timeit.timeit(lambda: sess.run(tt_svd_op), +# number=1000) / 1000 +# print('TT-SVD for tensor of shape %s takes %f seconds.' % (tens.get_shape(), +# tt_svd_time)) +# logs['tt_svd_time'] = tt_svd_time + +# round_op = t3f.round(one_vec100, max_tt_rank=10).op +# round_time = timeit.timeit(lambda: sess.run(round_op), +# number=1000) / 1000 +# print('Rounding %s takes %f seconds.' % (one_vec100, round_time)) +# logs['round_time'] = round_time + +# batch_round_op = t3f.round(vecs100, max_tt_rank=10).op +# batch_round_time = timeit.timeit(lambda: sess.run(batch_round_op), +# number=100) / 100 +# print('Rounding %s takes %f seconds.' % (vecs100, batch_round_time)) +# logs['batch_round_time'] = batch_round_time + +# project_op = t3f.project(one_vec, one_vec).op +# project_time = timeit.timeit(lambda: sess.run(project_op), +# number=1000) / 1000 +# print('Projecting %s on %s takes %f seconds.' % (one_vec, one_vec, project_time)) +# logs['project_time'] = project_time + +# batch_project_op = t3f.project(vecs, one_vec).op +# batch_project_time = timeit.timeit(lambda: sess.run(batch_project_op), +# number=10) / 10 +# print('Projecting %s on %s takes %f seconds.' % (vecs, one_vec, +# batch_project_time)) +# logs['batch_project_time'] = batch_project_time + +# project100_op = t3f.project(one_vec100, one_vec).op +# project100_time = timeit.timeit(lambda: sess.run(project100_op), +# number=100) / 100 +# print('Projecting %s on %s takes %f seconds.' % (one_vec100, one_vec, project100_time)) +# logs['project_rank100_time'] = project100_time + +# batch_project100_op = t3f.project(vecs100, one_vec).op +# batch_project100_time = timeit.timeit(lambda: sess.run(batch_project100_op), +# number=100) / 100 +# print('Projecting %s on %s takes %f seconds.' % (vecs100, one_vec, batch_project100_time)) +# logs['batch_project_rank100_time'] = batch_project100_time + +# if args.file_path is not None: +# pickle.dump(logs, open(args.file_path, 'wb')) From 65c4a74bc218fb9357f045e6e730ca3e2c18555f Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 1 Dec 2018 13:38:54 +0000 Subject: [PATCH 155/233] fix the rest of the timing --- examples/profile/profile.py | 157 +++++++++++++++--------------------- 1 file changed, 65 insertions(+), 92 deletions(-) diff --git a/examples/profile/profile.py b/examples/profile/profile.py index c6e73eca..7b449acd 100644 --- a/examples/profile/profile.py +++ b/examples/profile/profile.py @@ -36,96 +36,69 @@ print('Multiplying %s by %s takes %f seconds.' % (one_matrix, one_vec, logs['matvec']['wall_time'])) -# batch_matvec_op = t3f.matmul(one_matrix, vecs).op -# batch_matvec_time = timeit.timeit(lambda: sess.run(batch_matvec_op), -# number=100) / 100 -# print('Multiplying %s by %s takes %f seconds.' % (one_matrix, vecs, -# batch_matvec_time)) -# logs['batch_matvec_time'] = batch_matvec_time - -# matmul_op = t3f.matmul(one_matrix, one_matrix).op -# matmul_time = timeit.timeit(lambda: sess.run(matmul_op), -# number=1000) / 1000 -# print('Multiplying %s by itself takes %f seconds.' % (one_matrix, matmul_time)) -# logs['matmul_time'] = matmul_time - -# batch_matmul_op = t3f.matmul(one_matrix, matrices).op -# batch_matmul_time = timeit.timeit(lambda: sess.run(batch_matmul_op), -# number=100) / 100 -# print('Multiplying %s by %s takes %f seconds.' % (one_matrix, matrices, -# batch_matmul_time)) -# logs['batch_matmul_time'] = batch_matmul_time - -# norm_op = t3f.frobenius_norm(one_matrix, differentiable=True).op -# norm_time = timeit.timeit(lambda: sess.run(norm_op), -# number=1000) / 1000 -# print('Computing the norm of %s takes %f seconds.' % (one_matrix, norm_time)) -# logs['norm_time'] = norm_time - -# batch_norm_op = t3f.frobenius_norm(matrices, differentiable=True).op -# batch_norm_time = timeit.timeit(lambda: sess.run(batch_norm_op), -# number=1000) / 1000 -# print('Computing the norm of %s takes %f seconds.' % (matrices, batch_norm_time)) -# logs['batch_norm_time'] = batch_norm_time - -# flatinner_op = t3f.flat_inner(one_vec, one_vec).op -# flatinner_time = timeit.timeit(lambda: sess.run(flatinner_op), -# number=1000) / 1000 -# print('Computing the dot product between %s and itself takes %f seconds.' % -# (one_vec, flatinner_time)) -# logs['flatinner_time'] = flatinner_time - -# gram_op = t3f.gram_matrix(vecs).op -# gram_time = timeit.timeit(lambda: sess.run(gram_op), -# number=100) / 100 -# print('Computing the gram matrix of %s takes %f seconds.' % (vecs, gram_time)) -# logs['batch_gram_time'] = gram_time - -# tens = tf.cast(tf.random_normal((10, 10, 10, 10)), tf.float64) -# tt_svd_op = t3f.to_tt_tensor(tens, max_tt_rank=10).op -# tt_svd_time = timeit.timeit(lambda: sess.run(tt_svd_op), -# number=1000) / 1000 -# print('TT-SVD for tensor of shape %s takes %f seconds.' % (tens.get_shape(), -# tt_svd_time)) -# logs['tt_svd_time'] = tt_svd_time - -# round_op = t3f.round(one_vec100, max_tt_rank=10).op -# round_time = timeit.timeit(lambda: sess.run(round_op), -# number=1000) / 1000 -# print('Rounding %s takes %f seconds.' % (one_vec100, round_time)) -# logs['round_time'] = round_time - -# batch_round_op = t3f.round(vecs100, max_tt_rank=10).op -# batch_round_time = timeit.timeit(lambda: sess.run(batch_round_op), -# number=100) / 100 -# print('Rounding %s takes %f seconds.' % (vecs100, batch_round_time)) -# logs['batch_round_time'] = batch_round_time - -# project_op = t3f.project(one_vec, one_vec).op -# project_time = timeit.timeit(lambda: sess.run(project_op), -# number=1000) / 1000 -# print('Projecting %s on %s takes %f seconds.' % (one_vec, one_vec, project_time)) -# logs['project_time'] = project_time - -# batch_project_op = t3f.project(vecs, one_vec).op -# batch_project_time = timeit.timeit(lambda: sess.run(batch_project_op), -# number=10) / 10 -# print('Projecting %s on %s takes %f seconds.' % (vecs, one_vec, -# batch_project_time)) -# logs['batch_project_time'] = batch_project_time - -# project100_op = t3f.project(one_vec100, one_vec).op -# project100_time = timeit.timeit(lambda: sess.run(project100_op), -# number=100) / 100 -# print('Projecting %s on %s takes %f seconds.' % (one_vec100, one_vec, project100_time)) -# logs['project_rank100_time'] = project100_time - -# batch_project100_op = t3f.project(vecs100, one_vec).op -# batch_project100_time = timeit.timeit(lambda: sess.run(batch_project100_op), -# number=100) / 100 -# print('Projecting %s on %s takes %f seconds.' % (vecs100, one_vec, batch_project100_time)) -# logs['batch_project_rank100_time'] = batch_project100_time - -# if args.file_path is not None: -# pickle.dump(logs, open(args.file_path, 'wb')) +batch_matvec_op = t3f.matmul(one_matrix, vecs).op +logs['batch_matvec_time'] = benchmark.run_op_benchmark(sess, batch_matvec_op) +print('Multiplying %s by %s takes %f seconds.' % (one_matrix, vecs, + logs['batch_matvec_time']['wall_time'])) + +matmul_op = t3f.matmul(one_matrix, one_matrix).op +logs['matmul_time'] = benchmark.run_op_benchmark(sess, matmul_op) +print('Multiplying %s by itself takes %f seconds.' % (one_matrix, logs['matmul_time']['wall_time'])) + +batch_matmul_op = t3f.matmul(one_matrix, matrices).op +logs['batch_matmul_time'] = benchmark.run_op_benchmark(sess, batch_matmul_op) +print('Multiplying %s by %s takes %f seconds.' % (one_matrix, matrices, + logs['batch_matmul_time']['wall_time'])) + +norm_op = t3f.frobenius_norm(one_matrix, differentiable=True).op +logs['norm_time'] = benchmark.run_op_benchmark(sess, norm_op) +print('Computing the norm of %s takes %f seconds.' % (one_matrix, logs['norm_time']['wall_time'])) + +batch_norm_op = t3f.frobenius_norm(matrices, differentiable=True).op +logs['batch_norm_time'] = benchmark.run_op_benchmark(sess, batch_norm_op) +print('Computing the norm of %s takes %f seconds.' % (matrices, logs['batch_norm_time']['wall_time'])) + +flatinner_op = t3f.flat_inner(one_vec, one_vec).op +logs['flatinner_time'] = benchmark.run_op_benchmark(sess, flatinner_op) +print('Computing the dot product between %s and itself takes %f seconds.' % + (one_vec, logs['flatinner_time']['wall_time'])) + +gram_op = t3f.gram_matrix(vecs).op +logs['batch_gram_time'] = benchmark.run_op_benchmark(sess, gram_op) +print('Computing the gram matrix of %s takes %f seconds.' % (vecs, logs['batch_gram_time']['wall_time'])) + +tens = tf.cast(tf.random_normal((10, 10, 10, 10)), tf.float64) +tt_svd_op = t3f.to_tt_tensor(tens, max_tt_rank=10).op +logs['tt_svd_time'] = benchmark.run_op_benchmark(sess, tt_svd_op) +print('TT-SVD for tensor of shape %s takes %f seconds.' % (tens.get_shape(), + logs['tt_svd_time']['wall_time'])) + +round_op = t3f.round(one_vec100, max_tt_rank=10).op +logs['round_time'] = benchmark.run_op_benchmark(sess, round_op) +print('Rounding %s takes %f seconds.' % (one_vec100, logs['round_time']['wall_time'])) + +batch_round_op = t3f.round(vecs100, max_tt_rank=10).op +logs['batch_round_time'] = benchmark.run_op_benchmark(sess, batch_round_op) +print('Rounding %s takes %f seconds.' % (vecs100, logs['batch_round_time']['wall_time'])) + +project_op = t3f.project(one_vec, one_vec).op +logs['project_time'] = benchmark.run_op_benchmark(sess, project_op) +print('Projecting %s on %s takes %f seconds.' % (one_vec, one_vec, logs['project_time']['wall_time'])) + +batch_project_op = t3f.project(vecs, one_vec).op +logs['batch_project_time'] = benchmark.run_op_benchmark(sess, batch_project_op) +print('Projecting %s on %s takes %f seconds.' % (vecs, one_vec, + logs['batch_project_time']['wall_time'])) + +project100_op = t3f.project(one_vec100, one_vec).op +logs['project_rank100_time'] = benchmark.run_op_benchmark(sess, project100_op) +print('Projecting %s on %s takes %f seconds.' % (one_vec100, one_vec, logs['project_rank100_time']['wall_time'])) + +batch_project100_op = t3f.project(vecs100, one_vec).op +logs['batch_project_rank100_time'] = benchmark.run_op_benchmark(sess, batch_project100_op) +print('Projecting %s on %s takes %f seconds.' % (vecs100, one_vec, + logs['batch_project_rank100_time']['wall_time'])) + +if args.file_path is not None: + pickle.dump(logs, open(args.file_path, 'wb')) From bf766c7d57638410f23f481a5d4a9b87c03d4923 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 1 Dec 2018 13:40:13 +0000 Subject: [PATCH 156/233] fix problem with arrays of non tf values in name scope --- t3f/shapes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t3f/shapes.py b/t3f/shapes.py index 8b65f12b..1107ee3a 100644 --- a/t3f/shapes.py +++ b/t3f/shapes.py @@ -212,7 +212,7 @@ def clean_raw_shape(shape, name='t3f_clean_raw_shape'): """ if shape is None: return None - with tf.name_scope(name, values=shape): + with tf.name_scope(name): if isinstance(shape, tf.TensorShape) or isinstance(shape[0], tf.TensorShape): # Assume tf.TensorShape. if isinstance(shape, tf.TensorShape): From a229e04801689909b9aeeb60c8c2eccf31210cd6 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 1 Dec 2018 13:58:57 +0000 Subject: [PATCH 157/233] s/something_time/something/ --- examples/profile/profile.py | 56 ++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/examples/profile/profile.py b/examples/profile/profile.py index 7b449acd..f2c3cc0c 100644 --- a/examples/profile/profile.py +++ b/examples/profile/profile.py @@ -37,67 +37,67 @@ logs['matvec']['wall_time'])) batch_matvec_op = t3f.matmul(one_matrix, vecs).op -logs['batch_matvec_time'] = benchmark.run_op_benchmark(sess, batch_matvec_op) +logs['batch_matvec'] = benchmark.run_op_benchmark(sess, batch_matvec_op) print('Multiplying %s by %s takes %f seconds.' % (one_matrix, vecs, - logs['batch_matvec_time']['wall_time'])) + logs['batch_matvec']['wall_time'])) matmul_op = t3f.matmul(one_matrix, one_matrix).op -logs['matmul_time'] = benchmark.run_op_benchmark(sess, matmul_op) -print('Multiplying %s by itself takes %f seconds.' % (one_matrix, logs['matmul_time']['wall_time'])) +logs['matmul'] = benchmark.run_op_benchmark(sess, matmul_op) +print('Multiplying %s by itself takes %f seconds.' % (one_matrix, logs['matmul']['wall_time'])) batch_matmul_op = t3f.matmul(one_matrix, matrices).op -logs['batch_matmul_time'] = benchmark.run_op_benchmark(sess, batch_matmul_op) +logs['batch_matmul'] = benchmark.run_op_benchmark(sess, batch_matmul_op) print('Multiplying %s by %s takes %f seconds.' % (one_matrix, matrices, - logs['batch_matmul_time']['wall_time'])) + logs['batch_matmul']['wall_time'])) norm_op = t3f.frobenius_norm(one_matrix, differentiable=True).op -logs['norm_time'] = benchmark.run_op_benchmark(sess, norm_op) -print('Computing the norm of %s takes %f seconds.' % (one_matrix, logs['norm_time']['wall_time'])) +logs['norm'] = benchmark.run_op_benchmark(sess, norm_op) +print('Computing the norm of %s takes %f seconds.' % (one_matrix, logs['norm']['wall_time'])) batch_norm_op = t3f.frobenius_norm(matrices, differentiable=True).op -logs['batch_norm_time'] = benchmark.run_op_benchmark(sess, batch_norm_op) -print('Computing the norm of %s takes %f seconds.' % (matrices, logs['batch_norm_time']['wall_time'])) +logs['batch_norm'] = benchmark.run_op_benchmark(sess, batch_norm_op) +print('Computing the norm of %s takes %f seconds.' % (matrices, logs['batch_norm']['wall_time'])) flatinner_op = t3f.flat_inner(one_vec, one_vec).op -logs['flatinner_time'] = benchmark.run_op_benchmark(sess, flatinner_op) +logs['flatinner'] = benchmark.run_op_benchmark(sess, flatinner_op) print('Computing the dot product between %s and itself takes %f seconds.' % - (one_vec, logs['flatinner_time']['wall_time'])) + (one_vec, logs['flatinner']['wall_time'])) gram_op = t3f.gram_matrix(vecs).op -logs['batch_gram_time'] = benchmark.run_op_benchmark(sess, gram_op) -print('Computing the gram matrix of %s takes %f seconds.' % (vecs, logs['batch_gram_time']['wall_time'])) +logs['batch_gram'] = benchmark.run_op_benchmark(sess, gram_op) +print('Computing the gram matrix of %s takes %f seconds.' % (vecs, logs['batch_gram']['wall_time'])) tens = tf.cast(tf.random_normal((10, 10, 10, 10)), tf.float64) tt_svd_op = t3f.to_tt_tensor(tens, max_tt_rank=10).op -logs['tt_svd_time'] = benchmark.run_op_benchmark(sess, tt_svd_op) +logs['tt_svd'] = benchmark.run_op_benchmark(sess, tt_svd_op) print('TT-SVD for tensor of shape %s takes %f seconds.' % (tens.get_shape(), - logs['tt_svd_time']['wall_time'])) + logs['tt_svd']['wall_time'])) round_op = t3f.round(one_vec100, max_tt_rank=10).op -logs['round_time'] = benchmark.run_op_benchmark(sess, round_op) -print('Rounding %s takes %f seconds.' % (one_vec100, logs['round_time']['wall_time'])) +logs['round'] = benchmark.run_op_benchmark(sess, round_op) +print('Rounding %s takes %f seconds.' % (one_vec100, logs['round']['wall_time'])) batch_round_op = t3f.round(vecs100, max_tt_rank=10).op -logs['batch_round_time'] = benchmark.run_op_benchmark(sess, batch_round_op) -print('Rounding %s takes %f seconds.' % (vecs100, logs['batch_round_time']['wall_time'])) +logs['batch_round'] = benchmark.run_op_benchmark(sess, batch_round_op) +print('Rounding %s takes %f seconds.' % (vecs100, logs['batch_round']['wall_time'])) project_op = t3f.project(one_vec, one_vec).op -logs['project_time'] = benchmark.run_op_benchmark(sess, project_op) -print('Projecting %s on %s takes %f seconds.' % (one_vec, one_vec, logs['project_time']['wall_time'])) +logs['project'] = benchmark.run_op_benchmark(sess, project_op) +print('Projecting %s on %s takes %f seconds.' % (one_vec, one_vec, logs['project']['wall_time'])) batch_project_op = t3f.project(vecs, one_vec).op -logs['batch_project_time'] = benchmark.run_op_benchmark(sess, batch_project_op) +logs['batch_project'] = benchmark.run_op_benchmark(sess, batch_project_op) print('Projecting %s on %s takes %f seconds.' % (vecs, one_vec, - logs['batch_project_time']['wall_time'])) + logs['batch_project']['wall_time'])) project100_op = t3f.project(one_vec100, one_vec).op -logs['project_rank100_time'] = benchmark.run_op_benchmark(sess, project100_op) -print('Projecting %s on %s takes %f seconds.' % (one_vec100, one_vec, logs['project_rank100_time']['wall_time'])) +logs['project_rank100'] = benchmark.run_op_benchmark(sess, project100_op) +print('Projecting %s on %s takes %f seconds.' % (one_vec100, one_vec, logs['project_rank100']['wall_time'])) batch_project100_op = t3f.project(vecs100, one_vec).op -logs['batch_project_rank100_time'] = benchmark.run_op_benchmark(sess, batch_project100_op) +logs['batch_project_rank100'] = benchmark.run_op_benchmark(sess, batch_project100_op) print('Projecting %s on %s takes %f seconds.' % (vecs100, one_vec, - logs['batch_project_rank100_time']['wall_time'])) + logs['batch_project_rank100']['wall_time'])) if args.file_path is not None: pickle.dump(logs, open(args.file_path, 'wb')) From 5535ce618d6485a80fcb46743ad03293420ea0d4 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 1 Dec 2018 17:09:42 +0300 Subject: [PATCH 158/233] compute cpu timing properly --- examples/profile/logs_cpu.pkl | Bin 700 -> 1566 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/examples/profile/logs_cpu.pkl b/examples/profile/logs_cpu.pkl index b896423052b8931cf0378bec4471f94934e5eacb..91c79369db3e8ef40a5035ebf0c499509b95d12e 100644 GIT binary patch literal 1566 zcma)+&1(}u6u`GhliDUvtH>DrBEH+j3WO+Ad} zAR^+?iy-J9kb_=4N>gwS2|Ibi?>Fzwd)r7-P;y$I zq9}8iluW(mRBsz4qC42WGoH^w8-+wpYiDsATLvU&wG?|U6T@+#eI}!IvQ-^dDmp2% zaB6z$ea{;EOVQHo$5kr0VDvUMgFjrQ7T~cb6EhUaEPp0ZsNl zsk>T+?IosD^%l&UN1$_>t^JSZhk}#ZRmXx1g|3{LoHf~5OlE)fRpT2?rp!tyyV0xF zT-G&32R}L?y1O_JJrsIHKKWJ%`5UG9o6z@WQi<|=86sZi%@nqXBJ=5TDB3C|>O+<0 zL|atE6R|L{ybAPF7!dgnMj9c1n-qULDtvyOj`I2C->P6zwr#_K9TWyd#pRFO;NQhg zDa9_N{!PSc)k&vYa@=}sp0E0FKI%2z4OaQ+F(2nn28eo)d@?3o;Ja zvoS3jNh~uYQsY7E`1U0Bk3AbFkP|y*HXzMrDc#LIDfW4P7)wSd#+un>%p#OLq>vXS zZUEU(A%t)my-i9!x_WVlF!1Cay4746JPxCbBDkyDZv-fr;9O#aEFjqkOruMnyY&Ai z=Sne3SBhdHB_|>qCi{MSxh)N0DHAPGiq|c^0GP*pu#=6)qHwBIh?p#3;`lB)7WMv9 Y_lLG^y2ZfUYHW-wBYlgQ%!aPkzsIb*=Kufz From dbc870b83d6dfe2bd832de40de08ce1652cfb8e8 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 1 Dec 2018 17:34:04 +0300 Subject: [PATCH 159/233] fixed gpu timing --- examples/profile/logs_gpu.pkl | Bin 699 -> 2426 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/examples/profile/logs_gpu.pkl b/examples/profile/logs_gpu.pkl index cd782d2336bb87da9e2f379340a5ffb80e56c35c..a94392ddf42ec5482262c0b8111f2704f85efcf6 100644 GIT binary patch literal 2426 zcmb8x%TE(g6bIm;ybDTw;EVbMq=;hif%t-oXzLvcbzw5QaTp3y?Myk-J6IBL(zsM& z-099HZd@1^x*&0(#OR*5c0pp?yYLV2-f3r4hQypEeauWJ^PS(FbJ}t}11Y9-NRqT* z7w%`8Q;?~;@z{*o4}h|H1KmQjZ0dmlraBH4ZGmX8w(DFaFTDk4@c?b?=u?J4Rv#kZ4BCD z2CvNEwLGuZV(c<;BMazAmqU)xI%WI~ZRXG#5h4tB__XC$2~mrn?ZnV_ zk<9A4grx0GM}o9=e)(#KrbbOC51kD5_`Xl?`aIuWjBnrLMk(L5Qil}omwyJb+vWqk z1P?Ii@`cYHjd;Q+MtG1^zS&scp>!&kpnJ0xY!ePMIO4l9Zw1R;*`pZOF_P+8YCz`J zEx!)rCR%zfuhE=kWo_tT(Cgc#fB3fJ7+W7Hf48uAT}-Y7Y-0HEj<#l@pTP;=^<{KX zz(h9;U|c83-Hi}-(wveX23-F%!(vv4K?bLN-tx;ej~B=AhKTe5%{#M2D~MrS>(a9f z&iSzEZsBrsJCA`46RBOt-fI`+^+0ZI{OZ*qQeD*%1{XbDA|4$ct?4ddbeBnaAc9@h zEAm=ESBp)o=T!!y9&9Y7iqzcL#xSr1Nxc7zjqTbNv5B9yYHahiX2Ljw>prejs(82w z4DJSzevOBb^W$b(soOF(O4Z`xEF>AEeA$bS?q37ZrMECLLMm_SvC~7AjR5RVY?c_i flMK>6&W>()u}xuc)1;+ovIQAiM&8(ra|V9{@T>f( literal 699 zcmZ{iOH#r>7=-s+1XtAmeq6u{Z~!U-Dhh!Lf%7{F83e1$Y_jO-`nr2wHYb=Z=Wk8F z{#+fJ{&3l^`rTnWpPg{_3JU;WXCZJ@W&%di*?BoXb;qymy02QAw8R-u4rzdz$soOZ zU$^V3=jttqkmwYIAd-W=J6_sN70M(;6BkM`BXQLjA=+bis6w5DkzA}IJfZxm4UW&b ztXgsjgMe&;tSP0CWJh z$T8flwvVp4M?|>kEzB{p;Bdji^ZUN(cWt}v?)-ID O2U7G5qjUo=0Tw@mgS;02 From babf8120be4e3647f859f7403ffbb38dca38cf71 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 1 Dec 2018 14:38:52 +0000 Subject: [PATCH 160/233] update the picture --- examples/profile/results.png | Bin 128971 -> 36989 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/examples/profile/results.png b/examples/profile/results.png index a6fdf24ad09d0a6fc038a3d1d90afda46686c981..eee729dd82f8090c733294200392b2c84053ba7b 100644 GIT binary patch literal 36989 zcmeFYbx<7bwk}RW0wGue1b26LO@h0->)`I11or^JEw}{;?iMV#GuSY=!!WoGck+FE zpM7rq>Q?9C)f*<)AfI12PiSpIZaaS|-B6oImva+$aBzO05wj{Uowy{D$@Ltl41EKf6e-#XN zii$%B+XQ4W$`v|JLQy;FZMu+uGFu2=?QY^Xg`tR-gclLK%gAU>eWmy6ZE&;KSbs~X z{CaJ;3Cf7`^r4FKQ!fsxkO9bH$%=@Pr0sn^;H3wnN9+SiIt1hL3XB|t`U96u`~@@B zmbxbVqFy7}z^_V2Q9;n$$${0>!pY2%)!V@tE2d|fToeU&uKeeKKzEGWf9UkiH+!VNfBx|@=FJJ>tA33`iA z{$*DXzWwJg8zuQ)D(-e7lsbwkh=VB-)F5MX2HWaH#yforh1`8c|pdb2pXQT;LTw+#tP zH*;4TXLlPXNAf>5P0gG<+(jrU;d1hS82@tz{=>SX+dn{n!@>4P!p6bM&i0?x-EFM? z59)s;|5E=mSWv~r+tOY~!p6bU(G7kHB9#1Ge194HmsIyZNjdoi{*wN&Bq-}(3b53+ zv2eHk*RX#HWGrm}*6^A9Z$p$_Z7ks!{F$u?B?l$jA0*lS0VVv-u>Hra`Ge2@{O8{% z`7dGoZ@B&q*MA9t|B~{*z3bm_{g)8-E2qJ4;7+M&|`j-OM{O+2Lv5 z3%8H58n56V|5p|f2ngf|vJxLOyqAtZ#xnZ1tv63zLBu+TM=dgq#%UH4p9fA*lSuI! zkjwW&c@U|_-bV|VaD=n$6h;YAOQL z!ru6j`ZpC-1^UO$!hwQel+<;+-1yM_qs2M^NcoP#Q4G|hvx8mjV7wsb{dh=7b#pQ6 zhSL~EgMnoXWsJe^S*7)N#Ph5eCp8?@(}@}c53BIE1a2DQZtAvz%7DPf&)HxeD|70m zC;G7V{p+oLfa<}gt>0~lRBg>EGV0W;JB|@5)w$EtNY88h1K9!_2>F-et7pf$4BLCJ z#Rhaa6-SF+xPigz8)Tqwcj4YXrPJ}Z>ljb4^zg9vYN9%k)i*SK&$(OVZ;vMUnxA@! zBWbV|l8SEP{eyO22MKFLRR`gUo3-bCz_)eXJiFv%@_7H}{8iO7(>Re$O$f+(P}nVo z814Ce$4x_2;8!h3ZZjF*GotNF!wYnbY2chpkvO8lhF6C5-WUhVtvgNU20Gr-D-uyLzCeTL+{DS3_1EaG_gCbGvB~`P8to@dykD96 zAZ{_@B%(0yO=PUF*=J>liByn}EVsx6Z$sWxn>ThP*{g7-V(y=V=0&TU8gK<57CS!I5^x2H>YYrp2n6B4xii_#$+?WSW; zv(i+gc+@ty$DuM@HgMv|!pb2whJ~+UB-EhJypy1xRc4Ei?$^e}kSl(!ypyBPoOD7( zRTpk)vWZS3i|-?=`>wNNoMO6F_}>|4=ANv^$SUZWit{}hrV_OCDSe;G9wZ8qH_zgb zjYZ>GcUheBUiTMe@V1Er%ghVPcS zc)a@(VP0D0oq)3I=G&`9NY4~!!}$Lv z^n)xObUBoFSoHGO3C7h=SOlD;i^ZitXSqfVw32aM2PRbgltuLMgzPc47rrv_!Uu%T zJ{w8^rT%kOtBo8ins}7&S<{r;?S$$XtsYzxD89{bod>Iz1w7JuRP#RiRT>xuw$>ID zdBM&`Y>Q~yn+>xWL-dLcM%w6niygJAkLr~){caO7d;|*ykPkIwTi0|M@29I1bg8*s zAJr|F&!@1e%{8g_Tlg5j?@3o3Nz0Go9F~f0fcnAN+vTw9$LNldte4lWtvwO~=dAAu zBqy~hS;c${Nkx1MPe7y^hLc7Xn@#88>}wi%H1F~zWP4>PPSvg zgTsh#y}y{TZD6i$+w#)BTv@YZGlc80Gi6XSdL~iy>y?a%zLs1^nqI(z?NN$xsq|&z zMOT?>Y?qEwyrfuJ=Wue6UBf}9Z?RGPjjV{RuE6(Qjj|v!t0U&}+5iLi*n|F2OOL+9 zi7mB1acMc7s;%X(xVYN5XXMa&t7~J`2z#_uulJQOwFDRnacQK(dh2?q_IGt`KBdfc zENMU|8SiK7+IKaW0I4eDS4SkL``c#?JIDL2`J49IT)NWY^+&)QTh%ys7yH~H@9AW4 z^%EZq3u2S>Q3oa%ylo|>D?O7M)bTe2C7a7lya_hM6F0rlD;x%~6Rc6tRusta@Rx@8 z@2(&~MxZ=fP8r?^)00TSUAY61=q^-hc_3(}u4jemZ3xS7FcR4iy=;_hvi4To$^2C$B+ zht?{{X}<%OaU?{D*tm_PtrH6@X%sXB)5qVuG+^bdj$m+aQOnJ2k+F9j$n49(q;=?o{OJRk^;!UW!hwcHWj?{d88t8r`C&xZj(Y;ur6hiZ#e{~ffRGW6lp8?Ac6HD%&?UW8 z`r~?CudBwf-~6dzRei@qSI&mw!2u(Ml7iA3XlP;g#+ts6;^^0*Q&HP;D$l~2wJGEL zeIlxxhkFG^+E)cf=Ee$16?vzoBx?Ayy!bnP8FJKOg}|9QV?Ov} z)OCwxIKith<##$|jnG}z@k1bkf;I1b>FAM~EnQ7DX>!fRrC-f6E9;Q)Jo2)f-|=2T zA_*gsc}QQ$iKBi9$zUsyu*}ObcsMxi*5g(YmrXG^?3HwhJ}Bo1c=i1p1eFY)306LJS8EPrlXmE9UL04X6!f(pxx(|XGzPE4RyKRo}3s?9Noh~7qJFT>0 zBXbJ}I{EG!B&s-JD=I41Ur9IeTOJkkB;EPev>1uOVB`slx*x?AZ-j!nUofA9jnTc& zbh%f9#PW-0lZ4pk3_@NLy&~*7^;@!UKH4jsPhf1n&J^~}@FgH5FfDC+ z7QZ|3@KrPb0f4;XbOK$YzVul+^|RR{HKYFOi|*7*g2e>&(D&j+q50z>!kwN$ zd?ed?2bYew^q1U|AhBP+f4}J<30z_2dx8bu-ksMOGYuS$UCzu3ezb5&&?!wV9*3wm zz4g33#raa>d!ak%sN9A3XALhnQ#*QiD7mIM^MCZ_Me$?{^g1|ysA)@%dmp01{7c3T zSzPR)8+;NHu|MC$=LH@VBp0A;un096XIs@%ZqtyhLOtibeO!vgwI8+`^2i)5^YBvA zHEQERqi3EwxQcFwM%6ySD9{9J>91a#?XN9?Yny4BDJkDh%3~=IWLk z_M;orA;vTFj=eq>r!-B2e&P_D8F^a(YXw!~80)wC(!>;K%)}`Tp4`sG$5=i`Pt@62 zyKi2K4x7B+mxsjvq#K~PYTqlFhHpBJ?F+hY#Hxmy6P|z=F4p{}aEsF15Kb-#o4Gn;qt-Uu;tc-7Hc=%y^_fYm*H|N=!|Jl=}(;3PoZk-__K7N zSFSu$%N{RE6vb!k`L&fo9k1@^8kj`y#DlrD1Z%7bais+=Q^iACL5E5fhu(z#{L@bW z=nd4-)x>lpX`~A2`8orVemR5J(sZ@%uY>{7kLzylO7^%2sBveu zOOQ~M_9I#Fq~FrT4A zHS0fq6df9+sf1_p)lM7yc|;CwMuX&?yi=e(xVB94&N|%OioH9|YDOR#;4y<4+~tL@ zKm?nbgrP8sk4^ggYG&dy%>2W4`>#Xxj2U2_V&b1iW>bWIQU{Ua$Go>;x;|+rj8mff z^Pz)cC?@5UVI?#P*+0UU15pAu`XOVE5oSFYt-)IDR7GbAD1S;Zc!ko}MlDfs z$mPgV`OqaLtIpK<;~XaYOWgEz8=_sw=wJ3ixFtWf6TN0&X?XjJuIJ#_F{Ci4_fpF9 zfolDtc*2Fd>}C%wV+$+n@e_1)Z{6lL+h z%Sq^J;`aZ@sbBr)A{II?H{LHhJV@K)kk~{fDL9;?`qUH>tR+C=9n&LBlSwrTCI7 zVsq#v(Ll=%!|!yrmknMV)F0cbQreZ`95kssUE6PvQI6Xt(Ip8)gEHKbELYN7B+~sU zH}H!{&$7hqwO$k0RqWnj2X7pwPCbwjkrf{w%BI|So43Yi59s?PNvNU&ShIQt zh|LlCY_3JDDA>n~dbLotDlk2qm^%j~h>XAKKiB0L>W4>rZwa%A@QnJWmmANB?B4Rae2rGW7TA7K zTCnqdJa3}PE%1iACkYFrz0gL$FV@{&kPhuGnw7|_3hTvS%c&lBscVVi8duE7uT3&w zS-$b_Z6I|FykX#T%o=U!>PObw^ReGqiZr$ng7N}C)Ax#C`WVfQdb4L6XB7^wuTS1q zH;Wn$7P(_68bC~^`G1E(O$2w9)VwB#^Cpn?T71IHnIiFX%~vPsMSVgmJY2sY8kW*j zXPE2HMP!q*S=_F)O*M^3lN$kWERO0k#XAp2@v_(YN>Yg3mY$VkdRr54mD*OVZyt{t zH{LF6EeMRqFP4@GG}pYKCFOd{!%$wX>kxtPQ_H1&Lwt14Jik`R9-JBHd1IE(SRN%5Fje&qKGc3FRJ>-gW55uBUnFen`q7_ z_y8mcFtA3)=(GkiLcO08TWVMwgEfik(>ORe6)Y$p~XUiBJXMe!6!m0ip!vuqbpj0vF?4_ zm3y&#_(G?rUlKX%Q;(F^ybr}v48LGy_%iVb#Bh@!0{H+ z6*B*BWHNt=V^Bl?Lj}+pr*O5Ptuc0;IQlWR1v#tY+S$Yy9*bTc39aRI0%DpKB)n>9 zb~aR8D7rOH83$b}O?4w`*M4kDSw{(tAvZO-n+nDA(DH+MqXN&cg35l^6jjCv1w9G% zbbWKMK1!6Q-TPe|r423K%+w(_wZ(Z?O*Zdt&r3v|FM<8uEXSIdxWE4}MV6m|wb4YX zBt2EH?y=WuYH&0YH!2E9@~h#U`q8IbYwo8*)mRkY&cFyL6oqx`B%*oQGj1rK%fNQE zt>0zMzX{tZh~#ID7aKmRn{Lvi1?7k5Bo&t1?=zD-Fo>u(`na%J-vW9NGRJD~GN3I! zXbx;;iKbs=sJBlcZ7TAO$>wZjZ=FoqzizuZWe6z8gTwO#=xcRN0zVHH9N&~w_kMg{ zS^6rIrsv_o%fRWPKmc%4i*FM5tSdfcs3>OFcl_68xc_E+?S*w%ML1uyEr-Y8)!Fgb zOI%VkE}wmN&Hd464DdEecev)PW>H)>HK~7G6HI?+EE%Y~Hq?8_9jC~ShX%F0bN=+U zBWz+>X-D5z^eJuXT(r1=D#Vc_iD$hnF|do=Ud5^2z^UhOzJDJ0qvzt5 zH!PO$hWkPp%m_zTNZI9^;J2na~uLNz=%_$IVH0XCt9+v>{ zkPaSlV4stddaR-@t9b)*;JCNtVc@tOj9?4@faF25l9)=OOix&Mmq|_hR}|dl^zw;C z-9Bg3NR9Iu^?I)JW4L8}Zmdq5Woq``I6ct$dFo9%qc*A&9qxundcN}FxHot>UL_sh~$b zEmtK|da0T}d1_jzX)t0Z*l{6ay#zTYmwsKkN%!(1p^+0u9f zKw{tKAwyPlr*DzaW)v(XX^Eq+P)sKJn-n<~MT#yLk+%35J}uJseWD2>Yb_FVL~#r} zr_36QPsQdlyh;}C=W4szcN;>>U!0*6-03X%1wS`Pr$aXAI>SnzzX`ey|McHb|2Nw zW35*{?aoLfS!T5e_Dv?U@mpo?F5|Mn?A*>iL+

YVn2(q}o42KW60fDHZ&vq7ArM zl5%nJ9WG8*^(yC|>lTcnuYH!rOfzlg6g~0g?gmim9x5K|j7^ScZ}G|26>22(+sg&28ZOz5p){#+ zP6d@5JNTv5I}e3Y{h|6<$td&7veo&weS_>iq{fdt1Nm;FieleO8WWJE4&EoEIPJmX zZY+QMD!^6&ja{(X<+Lx7*h_#X4OSled5eS9kiWe@;o*;?OMkQLRJOk1{jeCx6!YSF z%*xFdWrEH6rkplt%tDh7v_9rTKi&#H${YOFUy-%c1(!xj@s$RC7TYF@Tw{2MG2pJg zver#vv0Iv^k5t@vEF6pHJeY+TM+b0>DZ9*hKBi1`1Tf?3OI^24Pc8k_<;+M=z9Cq| z?i|DnYZ1Ci8=RRL72|7ymMhsFV4P{*o9D!%b6IJa5|PNW#4?uWu?8gjx(3E+5M=CT zij$p=-IZ``O6($t$o*3208#>WlBfHLE0P>#+H-E4iHw!;ps8i84-n$i7}v_qj4Ox& zBkK!%0h%V8fK2Jh+B&c1gwmqzWl#Tt<4;mkQ>YJ@qxcQGsneG#z0cDI2J`^5=n6mf zIHOm2DITZ_woSKYnyCeUUVvu9Ti2Hzf&s5+&sh-7QOhsLq;r zh8w>v5F-jh-K*Tz!N$P#i<_tzSa|;C!tARM6Tq-v-{2Jq@snLzdEJb}F8W1B|2Kh3 zi3w2OY_!V)LNs=!Rq=^RgCm6nWdZ^y-GyCtYh>eR9#?9Bi=QtgwsM-apTkkWew~C5 zf6FJW6@|GlDm>Yf9LRc&k;K>v&v@lgU*sS)yKd`pZ;J%Spb{}n4$#x%n{02rUEl7m z&U6&NW>Xc0kZoxN|Ckqcl7ztcy7xC9^-U8{%~7y%Yf?rlXQyrY4pf+Ub%VcR&I%E$ zHpWvMm{_@`FMmM4+;+Yh$}QmZ!IZJdvv$DixDF!?FmOSP0XwtDd%XRXr+b?59(y-@>ZQbz^Jqh}p6VYC0X#C)o9N-p#qbk8d z-0^lzD?5wA_nsyqPVr3>@s5rzLAW+R!iAQ0)4G4G>=5H(mL!I@KsC|ANI^uJyER11 zUB$5w%J^tK^&zSrS)#y&Q}kih@M&X>KVvZT5V}9xIA&=ttZY$RZ_zz%F4g7J4y^$j z?q&`JFT`o(?KWg{OSUm~F>A47Q}Yr3AXW}y$nw!FxIg>>edJfpw z#O?3X&$uzMv?rIhmTSzMZ*ic0($X4)j+&`?#*|||!^MZ)5o4pM0d4$^#sJ~4bTH&8 ziv#hermA|1pwMMuNvs=Ye&43*Nn^J84b;4oDBI%0d_#nGdF^2HxtQ4b`NX2wlTXoe z#rrhWCn%SgSjV@{Fp41weUV)%+LDUtADw~WSDvTp9{_mEQRn03U?Rk_DyZF89#TB zAj9Ti!Aad-njaWid$L~$56Zdw5eny3>{&Vy|7Kt@}=LNf;BEw$^Eo zm~4Bea46hTv#hsNQd8rIC!G}kH%wH3YPj1OXT~!vSgG4eJX&mh{W6o__BHj1>*I;A zo-X=e;hUo+z1oK3*+2O+JVnqiZ!VAO06Nv4KIk8app<9PPSp&}EnBg%6rG&rA;wW6 z6QZWcgQs9aUOKi?H}{q1d!HK0;|j>)&W_y>P~UkRI%ZaCL$`|vX^EtC(;mVm0)t2t zxi}O*VcuSh)a@_(2$iH`FXj5r#2ZHDi?+QA2@c5|?{lI=RyDPKtdNvp@z@vRu zj%nw;s3y1g0e6BWd@InOX|3{f3HL1R4p0v@uLF7@Vb?F?%_?Y$o5^s+arL!(#ck_^7L*0$|dQ<&~bp}<(Tz7|MwTDMPf}Pwclab zD}&Pah@<-)S|{}m6>WV*8S%F&Gn34I47K}qbcq+Q58_I+z4gjVg)%k+PX?Xdpc$-T zzLWH~mgQ0Q!zA5~PI%+R*_oxxCMs^?AHfz!YEdmIKEo}_T>gFM_^Q}CzQ1}i`C3|FfSb?$ExY=J#`++Z6L6_HP}wbz`-AHFs~#b` zx>WPUI2S7Rk8UoBZN|Af?i17+hUCsqiQH|~T*btl5~t%YF}Q|+CCYBF4~eyGQm>3E zhxJZsbz`tmQ50B!*GaF+5VbIjUt_ha^{(C+c8UgM%m8UQFmh?$w2JR|JuJn6C~=y9 ztJ-~AZ=A_;Y?IOu8}5G}p-%uNJ}GUJwBAB3zp!y^%-*>GV(KtDT3HKsGw6!9RoD&1 zEv-Y%CMLw5eo*)Qqq%iG<`j5N^nM)Yug)BtL*X|4&&~gKlEnYHJLk=Jq$UtodFzZokT*FTsr^7c1Tk`^N zUPRu#RA5oST6P(wM3^2lWeNVu+%q*i45pHE?+z3Ac3J9-T+|xpW$l7|_I}HpV>T9? z9jBg0WbE~IA#O)H#>mT7LFcL9Uh2033Kj16UhODD;8Z26FnULb#hcrga)Bp=S;S}g zyAjI=@g0$qyhc`yz+g?bB0pYfUdb@)v(}&b9`1G&j>gD6X$e%uem1Ag^9OS3Vy0z? zZIrp5l*%$v__mj7woUnmTCYK(f?sO*gveCPu6;QQkcdW-#}@O81GD+QJ8Ij#R?)M} z^`(8g=#c}e(&ANVJoBBK1D3GzX)$Jo)shfsh*>(`)GL$JHpIptNZil2+)&_w-?h2Qu{ zz>NyTYXs0R?4%`YsJ5?08o)l?rXn*|verySrMc#z_dl`EpV>rt)Tz$|=Ps$l47SD56BV8*XXFF%foh0JMeXuq%+*ew#O1K5GmNdt)}9JBo}twR90xv3 ziJUr2j}`-cyb?>E8O^GC1MEcYvG|fy&6eUas$hLctU8NQs8$uL*w_!c*x|@x#{0lT z=Z=K+J4+sZuT?ayW!|HBwoH0(Rb$Z(t$?dt`&8cKa9v;*XUex~*csiNb{PZl<$k0m;7m*J@`fb%Cw@|5?Rv^x*iO{FP^q`BU7%-PEq~1d6ThpOjkxLxJfb=yvgzdlwl3uD`8 zMVjkrtf=a+W(?DmZ+B%w^76 zE*&qSFf(QOEhQCtvwP&0UBf_h^!FBe(CG)D=3US@fV_`l8`=p<*Uw?onugKR`hcc zBbvi(=;%?@>J!LOv4an!Xh*J4x}5!ZQCHWefbM6S*XQ1}J>z+1!J)ukta@->Y!~u@ zGrjNq;I>myLrdHO* zevB3+)M8WPdgbK}r^?}iCunpypSeZ$tz56%dXaU;{U!Y?THd?dRCK?0-aPKxuy3ikD~ zET?9F;w|wtE_e#zAJUF)TyCoe11J2#1^097)(`i73!=97E2KJ0Gl_@gt}0CwH!RLf zDOU2xNQ<-OElQdtAdtY6I)O|!4rg0|Dy|7KKkQ5RROogB#S6>+p2fv6IDkVn#){gS!d-9El)dAldd)svGSG)_d=!}8MM zcwd5ck{-Lv!JmV+%@*VyCtp=o84L-1nScpl_+J4|246$Hyc{1NHdvG^h0mzdsZm0N zI=EkS1N}0WgHxe+dm9rL{ruk}~^(WF%B18`J~rYE`_RUeb6D z_FB8V#)=u`B*K_Hwj(#uQOo>olgHN!lCUcZCG&f{X+sPF03u(&^B&BaQoHYU{coEe z+~`*p<$|WAotlsn6q*A|HNkTs2L*3dry;WR_zqf?`4t>-ke}I@^-ocZR~7 zlg3Sv7?)g}C912lAwt#CqtnYvV?RnVS6L=D@I&-PnxAl9C<_H&lS21A$zo5asRA(- zdkm5c`AB*gib+AmAanM!>LTy^?Tma7!-Oygnn`4fB;VsYd52=)19~OeOGIu#>=non z%~{xL9^8w+&N)utKU`jPn+|sRHY*v6=vi6CBFb8cQ@Qi{xU?PV1o?du11Udzsb(vf z_2W2w*}4x~{az>{^Ro*j7Z_cgmuXT;rhJB>NK$4utR`UAGc)_%bmyTQ9umr=D2X>N z!|6%90H5-F@uIz8qX6Wqx~H$X6dlQV7g&A{TlqAKVE|Fil3M~{XS=8)0DMXJJ65iym@zLbIJ{EQi*sKXb|ej zow!hWh&`CmDrL&i!UZ6>{P-aLwmsXFvJT1Dl)*7ISw>o{*hnsqnJUQb$jG;sLy*#x zGFnCFV8GJ25NQ7WT`Mn1_t^*qTQmvi;d@(Y_Hj{xx#bretG7v8GJGV_MK>9Q&iXW% zt1{{u*3tmp!3IHDvuIkqynsX5VWyqIgTs>7lD23rEkEt74p=P?9UFc&H@|9?Cj=X7 zhP{Ctlcwjw*&H3@#q6F2llrJNH=+eED@ z{s*D!SImuCuvBSTDpOewZAFhy!I;5(!wU|x=;FRzL))cFEp|LN96Tzfj-#R?94sY} z(PEN7q%Jis;u!2qc4~wL3neTo^(Ow4>WwP0f@`;&=-#m8Uqf@=es0CEi$e^h+^=ZD+QTJ*`m87H|@(1ZsCQxU< zgLU7Hwy++q(zZlJ=7Vn>b+jxy_H~OZ6c*MM7_>t8G6Vo|2$NOM%r`veNs>kPKugs@Oyht!`MzWq>!!9#v*%TwYtBqEI zEv~bMgPpF;qp#k+qN&{>)E$%mo5|%#7YEeWo_Lj(@HUK6)+E8%i%#4PE7walZ#h~% zlWLM|Sab%PC{N77%*~JCr8D1}TSkaq8;YfU8rmi21y!^EDv5Pe;tL@ddeEWDyjr`( z&!kl{_jX0mtrtC+jDi%cM6lT6)lmo)=PR#kto(0C#`|_VU$cw9ez+U?1q`%WV)(H6 zdyVVcTP0%)7N1iqPfs$K+D%?p>dzY7&EG0w6f=<%#&>tOrcDKdfm?x2K?8OO(Nq_4 zqj@clH?N<`L2t9t@?bwQ>+$z}H?HnIzvo0~vmz(3zn%G5*2X;%w=CVQLUaYmyY;gC2IJ!pz`0-;L`0Ak#QQ>J-hI^wD9{Zje@sXOzaN!_q6TK%zVbQDI# zwfq|!6bwEzVVIN;vdU*h+9QcHhu(l!#(1~ve7u&v*}4>VY67P?b8dOYho8O9(gmNv zN>>x_q`--+JV|#M&YBMLBBc5)W!_uall7xYY?Cw{4B7Y|dLga{4Ddz$3a~i)k?~o86PSyo$ z-x?rWQ7)ag&>544+8Ql7B(|nfDhx%_GL2LKXQRKT@|>VbMuA#`0jWl?;br;CmuWe@ z+1c8@gV?2fq!O4x6}OxQa1^&LD|ODNXjVBOS#Ab?pw%GP#+E^^Z zaA|f>Th8J|A00N$jG<`+syYGLsg7T4E^G4|*4c2$FsWewRn#t>y7dyPdOD0ujLeX(PCD4d!xL_osIdF564^yAyaml zf1QA)<)bA}C+ps3EbmJiJAu~9XFj@PXuRcAN}1D>-}+Un%9kN!B_sL}r)iwyzL;A$ znZ83Wd4apgCZjf)Sd}H1i%4{41v(}cer6M8#DXaynj>-`7tfM0ZBJ9)`k13fR-DB# z&L#FuPfHABXY3byapNH*eVUTHs(w62Gi)X6q`lmbmlI*!$&g{|LSIj1>m04qUO(={ zv#2R2Le%x+KzjgdtT-$A^1wB2?de;3HEuX#nd4Bp{KoNU32+Si(%!&hPK&WbQaz); zST2scyeA#c$zAlPG(*u=d(I#JUC5h_02<)`h34JAT6CTrK{w#1N8?+Ze>U>6!4Erp zpWy$8`MdYmzWX0dMK}rYA4>nxaQwsg`Tr9|Np9a8`DH$eg)EJ|fQSDh(jwdWCicHm3g7O_;v4BO1zh`SB=U*r#ETaAwyT z4@X5LYa!v7_Z$;TkD_m6)bM=MA$P#@f*Nle$5aW;tr9f2c`FoST=}bYX!9c9d^X#8 zt2AmRM%4fLxS1BrR-u0V(r`Dj=nFO=lYw@4uR%)KT=n^8Du4SKWh8Re)g8gMx{{Hg zBy{=(qR8=0KFXIoHij7*qvb3Lu7L6A>&%zL(#p>3->|W;(Ts)SCx|Sm-J{JL+rtD0 zmf)=JzVH0ICUa5^5J6N)o~&kfBx(Pc&zUUVC7X*bT*dp)TYj;w)C_Dq{_O}ZUgsny z+W_X511-J_j1+zy`=co?!|4?J0bIJCU%Cufg7-~*$AKHsPMuGKkj6=GH_|vqm_Hi` zj*>o#5ykj&6~%z?lur^S=DhQdg+vjfZvsvW*6|kt+Y95^0|Zr(|1L!OCeOQ}knq8Ge%8~(ofwwh=Tq}VQUqnpcEYlE*)Ck z1ONk(_r{aBwf9Hk<2bWzBk@Txr7QS{pfMX|WRArr=4qs3PdJ@A>CG|Fg@>=@tw#m! zzKbs9#BbuInv7@#_}*VW!(=f>(j4RQ|FP|x@FJ`zEh;0a0-G4YPWPpfL*(robrh%Y z1ODR-SJs%Yp-JTiTPmyT`i&9FLd7thk^q1Z6F>G>Pb)To^sYzLp^{n;_6deGS5hSD z#4}MLvB-v<7t(H_OppM>hgvs~L*FfrZA8r*%T@O^=B50yp*Euk&^Nm3i-kV+k_l;s zsAjsD6=^11fO(Kc9 zptoML1o#EwrtYUin-AdzkTiXvsM4~m){dAM{M@n7=(ez$_I>v-9G@(A?c+`iea-t{ zv-FHc#^e7V{@Xb2KlyKb)E3KmWNke`Fp<%sD33$c7;1Uh_cM#Gu=Y2WJ-UOjlQ>=( zn2Bzh(MAJhLKI&P#}V+|&ifS6Gka_Fx=06kqT)3xa}pG2lkoTW<8wDOK&Arn7v*P> z>Q#okcWFK|A622P;?0d_Q6%hSu~o6+QdxV`A2Py=)8OPB)hfIpFJru}qn?E0KOoEa zJCceo%QHS8MM_|8Y959zX!_JX4$}T-@*V#-h$o!l%7x`6Hn_~LfP>15%FH6XvkWh? zy||>}ZwwDaj0mg6wxADgDe%9hhSK?a)xE4PNm)nFjvjg#lOpU2_~ka!_w${)z5f03 z=DDc9v7)(-y^fAq;F-Q&dpQ4=gN2DvuhopMFDxV+JUi{JxRifgOi#O$hJTZFmoumn zm$Rxcx2qzYDqxDm4uF6BSa<$n!q)9TNjime3M~tH0yR*R0)3|7F!u-ZK z5#OPX<(F-psUcsktZ?&q@GfD$CUTPjmC~H1)feqDrqu!06pJr=3p@wB0=!qyd5K|N3Fxv>4*8|&QgnO--Uu6sfw8m0Gt2y`i%YvC`$6?+w^b#!i% zFZr9SPTcmJ-Oh`dz<>E~3>?~IyNI4&n?4HBLvZR_z{{s_@RMjmP>|@)TMOSAUkXpn z9Q*54(9f7`I(DlJv)WEaVo>^KZ`H zVUwZ3WhZ_%pNoRE(=t%70|~dH&iK*)4V+?lyY; zSFgVA`q}a8@2;#@-LW=wJ&A<-9}C9_m9{v0Iz*1DCDN=xwM7ZZx@+Xc(Ll{IvF16M zu|@|Fx#Zh^%rDFY-}e{lW3dSg5l8#3GUoiRJ!;xEv+Bb7A_vgDMHRVq2huCV(F||D zm^b=#IzTtNc$+%@2J*1xKT+ijdCove8Tnoca0a@TBiKwxL9?cDdhX|oulVNP%pO~n zrME~;N}8%i%sb8?f|k?y^_40ZRcx)O{{_{)FI6p@ZO#)DGX+#*jLyo6Te z7%53Z5+#!F5u7EBuCEYU)OY;4bq6b@H-!mX-^YqdZPFR}=HH?x5A-1voD(7|XPNX` z&icL^qZ17qI=?xn=D^K1cf5Nh(hw`mKAoO=?MsF-bI2p%Rd$`dBXmw~ywZAsL`!a& zg%ZO>ioX&4I#?B+4}Eqwb z)sziSS(XWQ740IDgL%fdjnP7hARhnZ0hm;eg1H>$Gr zos1@>iq4zdRq!Gu4mP{5pU!)UjM@poD*}OxvClLG6-GqdF&d;}id;_5{eSoLE~+a5 zbafA`I|Y>8K`-%^RyP#c> zm3>CFccsqx<*nI81P8uoNsVy;#u;@zuQ2#vDiUV(B;dTDbz{*ofR58nH2@>urp&Tn zj}W(;o7XBcotu4XYMXw8iEAgU?3$O8kZq%{QpjKT3Q1G}^kH$GbGSV2eESgrfXNC4 zX^X@VeCq##U!xheNsRfv2%}M7!K8v%{@!dqreQ4_4;w{1jL68s7w&P!27M557Gg zMB9;ktnqzhB34i&a#CI-B@Ux_AEn?61P2qJnuNbN`uc^ZnVhHP=V7;9?CWX^g={Be173D_Jhs&>9tlSI}M44zmL z`{FGS4u~}DL-i~@kOfs6;`hzu>O3REXhZ1X$n!-*-ptu`=h%#E)CTjP#TL(oqe^ z_1FX>)VH-ICEeSYP!O0)UHwHgRHu#CMa;kt*8WvnIjBlPoF6ZpBO?8z#LpOC9k^%c zCT}TZ5GYvmX~73}-ld{WT4%y1f=XBxCVtKI(M7hM6Wl^x*DFzfi?LPv7UXk zr@{6Za`^o+~1%s?3nid>7tmk{rhy!&ziL3piyO0*4 zQ?*WO8S|LaK*1r?l@al0xGWt!!KY6Idzmm$B3CN0{i^x~lGSLtI-pgu7W+9N%|G(< z1Nua^a0m}+#2@Y)af49n_7eNwpf7E+qRO=K(&kHb>bMc2GJPXr9QD2oZ0NSx#94ze zBVa?XH5wVTzKdo@>Hw}#g#j1vtQ18cOf9@UVZI5>Mu%<*qY3DVIx5I|5qhv@H(3~( zWR0yJufSTc+dJzXVStEkMd~4B|8v4No4nGwk|C5euJ^bDuHi+T!jRDsB?XT|0~?W- z$Q_4cG4B_{O-Ey(uQ38-^74>lvTS4bN%DDbNs-YilWNXw^(>J(NBpi)VvcD1P@xZhR`=w6F0@H)w;i>4uesYbg&~4Zmu|( z=mkY^&Kt8`@0h(5#qi2_;4b7(^Q9-`hCv4K$vQA7)zjS(+r>t2Fs3Jr14NPuUIs`A z$6SpM#RqVQ`~0+>t5xT`zd+dibKsS5?tqMV_McMXGVP!e9H~I*K)Ne7+jYP|{)xP>YGHh4d zllt@X2A3iCoX_gnt0)DNw@+)Ob(bHh`l`6^of%Htvx6w}gATYHzv+V=;EHR}1~_1!IiT zsDBO3u+M|#>5$~?fR zMjKcxq_Bok=vfwlJT@I~fE#JWWP9_Ov)PuW$V9lTXnQfa$4;5{8E2R2+-JM)5K=;? zNipO&o-h`|34lw%VcQ+q=vz*I^4h*kTl4%<=dJrO`@yKNrU!aL#^s)l71!yX?QSc= z4AbRsxMJ|Jdrh3C2h_Z!Iqk2ykVmlQH9uFwj$}DZfuA2yPEOypGld3;Q|$w6=Q9LW z4Q$Y79D!-o8S|-ji=?wtAhIm=Dt;F+%kBly!_knDG(-DJuiX>Ig5_4S#Of$%#-B0{ zjKAWCEFu(ZzTjwfeLO}K8Z@|1n%3?YQoMGO5!v$h zU>P$dYGcPZ25BHF3DJ<1Ue+Ml(WuFOVnbeLA^cb%Y}(@n<%6PR-(NJM7BJj_s;1t< zdYo$h<*M0}Vu?XD+kv{t-e`;tZGZ#v8k0 z!rwI$tL|oUY-ciOFtO)m;lb5E;FwL zrGDWXLFZlM*t9mBgQ^{kkr+WLa-sZWlL|^d2D40g8u1yFVm&Oy#2*48Q(V~6MH7>w z{od1-JW>pbQ^fX8cXIKSUaJ6MGRs3ZTv|n}x&k=3nnZO)jBa{+mkIgu!~IBIDOsC# zGzZ(uwgrQ&6B2o)3b2I@?wSK>^uYQ0(Mhjo1>BuR=jU936^t_49wAMmsM%}U)-~-z zI+Vl-bon8GS-M9H(avGx(I!#JOU3+0_ZA#MTVoMiGHH=yZOqFAcEJj7g+#Ts*DxEC zx10Hb)suMl3T0eZIhHIVn(%}BNHNySd~Nb{^1t8e?)4QNLn|r>6RzjM{yRLvVVs#h zH=aGWLMbQPFSpL@RTUf6r2UH&GJdEqpfp(CNt2A6?>AqU^t$z+=UMz4y17`)VPHB&Rw;5GN*^mUqK#QobpghhH zX&8vEMG$AgJAF}SI;EF}iKP`OdYGW?ya;^u6*m@hc1?h0MmrEar2gDE@O#RI+>#9f zSv~;*tqMEjE{q9yt(lbXmut>aRqQ)P$FxX`j6I86c^{aJ0wIy;osxRhNb8k^B1s@q zl&Rm#B6^X$dprq&KZ;&|l9+tX{w%P=fCsNZ#uxy_ex-ek*_H&QT16M#ni z4HRq@q;0%qIG^udi;N8vc?hCHMewuGk?7qmK4dm&LJ~KgEurCoK|>j~a7JJRvNQQe zd#M;_PLCQldNN|8YB6q7jm0k9@m~g=2C$aiz1%vnS2b)@qd~`wTBL3S9cmjeXhv%5 zMzBFxj-8mct(T7c2Vdl)D!gg&sreS}jQ~o}h{?=m#k6mA#J6e>)F){bC?Vzg=dBUMF` zQCSktjsT7f-VKBDwmBy>lxdl^^R{oLvWO$9 zk{KptClaK?A`OjWE=7y{T99IB_z_4aPRWnsC! zezA8mKA?k^cw}wcgi?R{+u(wv=NdZY@F>zhUEmeVA_*GLRzvGetfUy!o^Fkx0y4$= zvT(JGe$$pQY@B{y`E8h8LxgIb-?-X@I%W38-$yvr?@++*>jID*ndy3&mQ0AcXI?BK z0n-+o<3>8TP%LX{$u+YVI$q8(QN=pZ$Ck;T-V-m~n?NHEY8gskNk10ZW_?{3(tLi| zKaV!T=~q6$A+|Les6pHV_i)JLip>! zQQbP}_D@|Btfl8Kx9;qYtsB*RYLS@VVKt@6?_XX5Kn9Ewa|Wh!a0+n+)nrRkTLg0C*{;KLsaFb{i_;5Qcr;Xwg|BEv#B%)8vn z&7seySA_oEYa1fdTq7*z8yU?)Gr@TX2xn&wi;WSqHuNEj8c56D0pBL(V%U#w$(ivV zNg|Rk9Q%jjph_h856_oTzd77v$mxV(|`9)O$V#-q?!t`w0iq-4*E<_ zlS?uuNiy>X##Snpq^P;#C#||NV|+X`l|%1v;BkF>bi8j0_#^3P-#3L>mg}L_sZaJ- z?ye9s8_uX9(OgF1HnViBu zH@fM1VR6+K@b=J+XH~@27@li}Ns*@4nM%xB%TlfhF5mx!ra8VBD24e2YEZlO#rzbB z;st3Pu!5yCAW09V>`!P$gJ7zpD0g7v1eJvf`Jn?#*O^B&;R#{*zYbzA=8ja5Nmvx( zg~Imoo;HamjqEUxm^j+QZM(-A2AkgVv6wSAOP=>6L>3WsEN*&^tA0mKeWl1u4`m_( zICCked5Pl*1Ps$~Qv}ar4o<>!@IKjOW0hQVb*e z75G+NhUhNwSoK#=%D1VOsg>h(cUQUb(!ls1slS3VX<;;Y79r4Fck+(%SpI!hWt^-j_`0N%p@#4xF0T>| z3r*os^yvm5{Z2=^FZdIaH*3pm!WWuT3I1V@EN1X|?H8(B2aWO4>5L+_MjqtQRWY}A zqIT?O{3sygo^@uQ9X%$7itM^K%PabwNcLgQ+N1KE+$jMzaV}e%_S}7QW1Viz?;~-- zIt=ZWz@?E9h`Vx(sb~s9b2OG~`{IBRzpJa5-27B%PArK>`7F5krfV=l{oT+`J&t{c znA6edabN6zO;$@tj*T1Da5;8fAbMW{4{auP=W?zv+>#u}%oE2Mf?xKmq6X6hT(Go; z=5%NjI!vpY&wk3y_fO*+$yG@DOQP<66i#5w=0*^^9^WKod(q*PLDb`Lp#D_^%YnzF z2CuO6bPdQ$B?u(&-pT)1v2EnjAA+d1s2j(!8#r4<)D|Etu=b9jbf2MI&Dv$NY(BU8 z=HdCpUGE9gk18WABR|3^P#-<$TaUE4GM%j|AkkoS;kbMpX@puL(j#*I*06J6trQ5I zC6J090D8;|$iMGL0FrE5a~1Dc9xw^^9?6~}Ia<$Z4YKo`y48CkZ%l2kByDgT=ADw8 z)S-_tfWyxEk#3Q=U;H>?uBniKAbF|js&ixQSIjL*ul`PToPkM*N3w?ijwwQ9$Pwl1@>ZLuyvSYxC>R}9>^vZxr`@Jwe;{?n? z=mFv9Nn)JK?*Z1%+4JQ%bZ;)o%0eknmF^ z;1lxHnzPlKMZ9q$FnSVv0w)~o*Yfz8Y5Jy@V9fsxM>k-QYkxhlLoe+}akK##!4V7k zAZ@NY42IB3B1u$mefY8e0SJd*kr+eHK(}MvDJSBw3}k%((M-oB(eyNh{ZGX#ov9S63$+x{O=lq;@kc zTtTFAo%>_D_%(u1q*L7n-aZynR3USVG*_m|KC1V7bhVIOwb05GX~d6Wrm(KNYwG+n z>i=4MJZ-AplDhXQPD>X2`3T%)p(n@Bj?6BiXKs}AH&ubXw=g(89)5C z!mOb%?f}hFbVNm^WlpYu`Bw!-s)_@xdw5{#)OGW!3y$rDOQy9cEeaFAE{SV51j7NT zDWjoKi)r^8u|4@bWNdaZ6f;ahoIsi^aJ?if5Z-Su2yj?0v}WTlHlg9Sm)Ym(!QuZs zI$jg*@a$~fAJH;F5~lRh*!3>f8q1M4BaaZl!iJT3#DY7p@Bq{0uUMqm_q9i`FdN$1 zt6&EnT-}%Q-|}31__EyDQspbrhH@dC8K$eJEedk=G5ra~8IrEdk?9tQr6cTzJYRsc zG||4PJKcj(_HE>sFwNv$L%~`2Ulq`$C%n;S(2M)fcelffJ)n9GOe9@Z4e9gXKWjAd ziRcbi?KD#$Ab!BET$J{wrB#E-b#E(|SsmA?R@Q+Lx`%?*ZS}2ump>-+%;)T6UQMQ) zXTN)cK0W&(jxdad!3gdha?az5!<-5=mHO-uR|7=Qh+VMWE_Pfo%Q|mo&Di3g?VR4V z@LDteLJxkdY3*36>O{7b9Y30>TW=3}$?`^r8WM1p1Z{V&R-&Lu-<0)CP5ZrqLNN>! z$ZRf#d~C=LguL7CsW516cy)P0vVdWIdm&vtVmd1<{o_ukq;smDDkYi|%(>#(VTvbWt3cWHdV{Qrh1)?fqp+)Epl!jX!CW`S^WWLd$T}fQ}2F~ou z=V(zSx9k!I<7&THx~J-*!=q9@W1wbQHE<0oz4$uMM-tsk`!b z&C7v~MWX~*2+(X-cTc4Q)tyXz%V#?XD+juoZ%=bJ97leqq<;mRG_O-=Wo2Yoc!0NU zB_oCLtnedg)87!}wcljZs{Ir?Nrmj)R{cw~Gy5F}w9P{bbg)J7#rkA1V*t& zf~eq`QV1Yn2e$MbH`!w(<6(ZYW||5T@pjhKI-i0PeT66^J>MJ4&s&@Mg^m)uOQw-f z{&^|IA3v~HT$7EOo0Ja+R6g?Qs#uQlp4@A=BRz8Z79MV;#r?0+wEnSu8umXb*Z;Ws z^slV-|EOi1Og*K(J>2K|9Ax#+gP6Jv5ben zzE0PS;(}*^#f~InZGL!sUu_$)Hrwm2Sms)6tI5Ss7pxn%O;W%^VBmSzoB!P8`$_2% z!w*l!okzWRhF5}4&%p(P-q+HLhB}u{d9AdwdkoF{HCufJ^5E-WJg3(!%p6PGc2oAR z(`D?7;?Ds2IRC&MOsq;(4i~Byv(H)5X4lMyjacC8L!|qQZ+HPj{*kr$2F-9o(|5MU zsmpN~3%EZIoRI>Ya;7Hsq)81^!?;p+xP^a%f2=Q5roMy?c!GN$kXB2Hk@=4|{){m= z&0b*}%C2X7YkK%xk{pk8IZbm9x}RK&cc2#gV7F-HZbdn&!~CN~pMCXtfXUzBm)0?1 z?7#lVff%4U-MmwSM$|4Go{Ji&>Bb$Uz)EmYmQ*3Qz?k}yFwHySfhH?g!%t88gUq-d%bVz;aB7X)V=Ki}1i7Cs zev!sn-QlzasI|$t#aP?Fl|L?Pr(T{+-|2o=75rIFc=@q;HP~}{Dss}n5>7fNkkRm5Xk6*mri+<@0=b;)W3_ z#96O}E-r6Gmy4Bff2`tps@HER?ikThHt8^oo{T4h-GP;O;-ZetUln#yNush7PNcEy z$WeIJCw8jF4Xn7(JhJX-tE2ZXiOzDh0IcSVUa+rw+A0&z_~WFqCj$|UU$RIZx;&k7 zs*d~ck)Xa6_K}Qo$NLqw$kJgax;}wv3yZStQlTZ}VUdnVX?yHX=eus77oT=`&_wKJ zF4jb_?a+#``jI1tv5wfkWB6KVa50wa0)>JuFzGZmc{Mv>YSAC2_!5sImcB9IrI*u? zf$T!P%(`1TR`#3k(@0%%qV5 z6!H*)pXM-K^A>0{NhN}wU012T0oP*eoZjXY=#+=$#C#2WoNszwd=55POv=G`y zhyK6wXrw4OfJ3$HP{rMvG~Hc@@Hg;}$_2~ZoiF);q5JdW-%33Fa^JnM0C6W1bgNX3 z$R5cUF85w${T3|Y;h84}yEaPk`gl;q2e0|HkW4dpk6p3W(U_QJ5hOJKvyw^NeAUT=QC11zVUdu&QFKgx5$gHWHbeV2)? zqszO?*C+WclAJ!*T*BC$<@Q&+iFcm_6QtR>BFifA?yCg}S6xo==ToYsrBy|T%p~Qc z%?C~?f8xX-e&R}4^nX2zf;UcE1`F^9BN3{fadxmph3Qmn>Gk2Sc6y>WK#KQ5WJ{pO zYozsUDob{;3pg)Qdu8-FRPU|3|MDC+L7a#8J^&DTOcRz(|{$4=~1| z#Kdr;MhLO~0@S*AcCt=+R^+e!pe z$D1hF;k>zcLfj~Gws95wrF+u=J=4o(iw9G3ld+hH|`~D&%o=SUdTIxsk9P1KdTZM<%!(tui#ZST% zVgX-$^8%l@c~-B4wmO=kdkM(LGK}BYtRX zz{=Bg4Q{ew^rSQJU}4Q^BP*DhB|c$jyRNI}{F#VS#E-}rr~yQINPV}}*5$d`*7s%F zAT!>oP9smE^eASOlMyz=j)0@lCwSkHi}HcsG)5%uWVfhl>k*HfeU-h7#RymXa{GZA z)fwiGKa#lU@LYe%4Kdh`x*2+PpbB|)s5 z2oHn|f8^u$vbaJ3wMA#x=HlHsU5zM{*}lONHf9=Uj!|>@AsN4M)?%DC&y0RKH!Y)S z^QtJ+`tgAuVXUB8AUn*r){E!tcw*=E*?d)e@>pi0#aq+S=MwP-9J{trd*hGLUM}UKA**wxyo*hV;fD9MMLo4s%3C;dcj%fb8oE#(jIP z%DudLBOQoIf|Y=Yn#|E&J007h?MzoUb9bkxA!TT2yL})W&V5GYxvEQ3mAfd8tDnrw z^bchvoL}*>VyuFd3Zni6mre@+)>akQ{SqHo(4mAm-Pnv`)6wUksVRr$IEWw4?oD=- z=Y)*sI;rXPZ#=8jujy^Vt_(rOt*#h>#+$KC7#@(~%YikV=vA9ArYB^757N+uC}}nc z1W@5o&AYrL&=0APFuN|paGw5uFgEZuS5KiuJ3_Sk^4)0;spQjN*v(HJPN|D1Cz8nv z=qb#+i6tZ}z%FV7*V!WXC=B4s(ew%tmLq9TXh^E>3FcYfe^i27M6{_?)4<|q=SgV zp0TOef@|xTN$>y+!Oyte>Ik4rSt@#s*lS@pTCXp0l&!TtQUcybl$CW0~96CzI>2ar%9cc_I2QIKo_ zT<1t~8em^}j+yg|FI0jeY6%iUI+1Jt?RuoR`R<0Cb|T>8&{@byL-J+mq|@a!BW$p- zr&|+v&knAT@XukJaAY|x;Ph>DuZZh$O?kMutM|d)st?+ZBaD^6l*`=-H}uA(_!s)1 zAwm^H?J%y1^V^0a*w7Ak!vK1x$9X^Bkrhm5s1?U928&Y-ZcY2y(^x7gCL!vY-z4dEflL%co^eek@XRMkqP*yjbFW;rIs1X&2f6 zc zi)YFv_evFHt}=kQn4`XcS6DHo!rBq&`S8p{&;UmT$5=A=K0jXTeIUFdl0*s>;Q46P z>NBZ;ga;zw1V3PGl52>^?e2kue{1IDsd=$vyY1Nie)rug9HkFpg8dsyr%ejL=*HG% zlEPG&W5@Mr>D2MNB4U=X3u&JX6z8kn+HNx z=Oth#)iZ-^A!*z880s|v6imYXs4Ki=iH3Y%2n#5Xd^bAaK~5*TU+RrT;nsS|(fv(C zxDpGOnL7IujF zed2~={M)Nmr-BWtpjjr=z!OqZ?E^?fLOux)dH2U~vZUnvevH-CvT&6-J!5_#;h5fs zSO%*)Jqd2f1Qvq!79%mO(Qo^IFq(4|KsD&RSD&WyN@?2s?a!DhTUIVo51_0HeowG+ zOcpj+Y{}QY<8*+un?@EX+Zi3vo(l^%V&(=Hg%;N!4iY|!d{vJn(B*@&Nu)oNq%cUa zQ`J8t#0QGn;r|0 z*igUeipiKHTtHXUnAftqVe0g`b?z`3T_kY3Ak!!PdT@ux)dWs8mNj!FnncAA^zpA^ z5d|8CKMVN3J{&-D)8B2;AU~g-y}P^$Af((ZV#FSUPNh9QbO*4YC+uJ>=sK@qn~XvB zNvE@T8Qho`*wP~|%J!whSsjV%5pT+=vPCd*D%{12RKt@$+K29;TwnC;!?Dn*;9uSe z0ZQrD5zrZ(5~|=6d@z=Ygt%W0Ar$r}Cd|p3T!4Ao{FC_QnR);5Y?GK|+P`tTL8v0Q zfbMt0#7`Jh<88gAmU5Fmj(Us8s;$0{@<;ymnCeyD;Fp%}^Um!4VD}vEp_{^a+haTE zkjWAE*i64zigAx_JvZfgBMY{tG{*+?bMcbl4R@7rWG1L(5BxO|>-FFu3oVD{yvJZhmTD1v-FfI3rszhUp=NLN!jM$jC`DOdZ%o9Cje-_cQth(&kGb4+>C zXm+gKn!$3Q!iEmA!3Me_nygz7xQ4l7n6ylxf8HSxk9fD%iJT+kvA8Xf4Fx9$Zl(zX zpK%IXBn@-QM8o9jH#!09j$7na3~o(?S-_4YSF29h-LEjIRTw)%nLdihFV z9|yrM?vmto{(Ao7UqPr54XVCS0HE?Ka(j2l8=mhoW!HeC|8eg{5I+u2D$&59mVI*k z%;GT0fApEEaQR5guOZOn2)LfcH%=b`fxaXgGPb{uE&rvc?4zL9asK>NRPnzep2!gm zHNhvuLU|xk%jpPjO8~2d|0@!$z)}1+lhu*r`G2RM{znKbZIS=K39S5=LbYBt@}2}5 zTsyeG0xzZt#oS0;{mwg11YLMT8ZQT##9%3pU1VBY4$X$w$63MFEbV_tt8ARcM&o|& zDt|H&PwpN2sbdlBB?xYK!OE{-^C(;BYdi8+?1SGkNG4ua3%7CRXdbsVYVH_08N^H} z;<|c6!Kj^*hZ#dM8Y4B`8)1b+Ou0)^=YxMqF6SebQL*Wr&XSO>QW1nD;@sH#Js6=IwS`up3-75xPJopZQb1{IXs z*pd3Rqq%ZMvaU>gz$r3=Vs%N9a&5V|?EQXcMuwh05Hrl{O8=a!*Dj`aY#cMM&otCH z@duASlo6B6LVJjxrE72Gm&!g3{g}8(Rtn9ChrS3UN1O|faMDSoUu(z#~FIYS$4(Du*e2H%h(UwT{vBij+| z21bynb3)fUmAoIR89;0xR?MjehR&d2A)t`gi-`#Jeb=N~B!G#{l=-Q!ff$v41b#^> zMd&gv!)Ts}A7Oa(O*ci76>#l9BI zgNdIQ4d*E@QV}D(q2?D}RK#I>&r4S1B&eb_A}|My6`RU;JKJmGWYW(@Nf-ku;ierG zS`RwTyc#uXeX_~19Bj8(zK{NCL^*`H_>p&SAra9o0a!7iHT;?SYlVYExkj^WrOhgo zOXHS|r%PMi|8vlAJoX_+kRe!=_7mt$W$&CBcIgP-3kW5h{yfUd`ymEd=g-6$y-O{! zpA50b*D~Z*aJTm#lCcHeWFq&ft=vVVlyoV6_fh~F1fU4AlvC$cVWk^~Q$>fq$vBMvN$B{h^=C#BBWjDc!z$S)~qz ziNJbf+}o-BlC8(=cgWmczPsE`n)QAUb6||~yyyjbQCexL^I83mE5w}1{jUwgM;?3$ z9o(gQC<#WNc$V~zis@mv?^%*^`7$ns;idN)#xXw_{mNcD>5S1u(TZYCTH7hLMvH7P z9aZL5**zF_>UPfeHh-lsQ@rd@;X@4=EyMm6MnCWV$FaoZ#Wr&Ga{G}&OqhO$Xd|sl z$9&z|p|JUEAbjEy?7AEsPfLD1kr#8r+Hx=0x6d3?K4uH0=_G!!8rbaL&zL*;WpTwO@ltf zlY~>hf-yjZr(ByH^>S&P&(enEG7ICiMsOQ?Oz!g4p(^&DvBmn|0Ndq+4Q2}&KygM~ zt2WQpnV`nlU_dwtID-JwG2INVutd#oosbAJu0rAl_j_d@^;9TV{frs@)Lur)8v$<`dMvj{wVtpr^1#;N1XwRGkkCi62)t4A?{k+$S90As(kQ( zMTFQ#quF@0z{KY>!so@0m@Sj-;oOchF(lK-mc56h%eWD1BuW`$AWI+PHQYG{Dz*n%xNXU+}CS{9LL9lSgq2 zAI!42?73RSD66r*W!qr+PFj0ldlg!Hcz`{c+qyEj*CDMGE(t;U1#rjqbExC$H%Vi( zqbvJ2tr|OI1;!y7b>cO_`l-f0V|242iO`wxU7hxc!G~C2#q@W1dPaAMC%;uY^3RC& zJg-=UE{lc|f&H!tT}@4f(|ijHYk`;01#!*#yuflBSts#e72)9DD%kGC61QBfKGgW> zzA#>k*l9}CLOdY}?4&;ir`Eg(-xvX$^&mm6I16V)^H=&|K-%sY_nX zOQ0qWr3yeJZNOrjEkl#Y#Pc8Z|DXN&S`~ry1nAhKiIOjKDT=rpJ4_(cz!>FV>3edzH#4H{CB6qr8 z`Cd^zTN0@yXZrTvz69%ZL0ZYSy8`p`R9c4(5crj9>F`yV6}rgRX9vk6FN zqvz>Mm_Gm#Kaj`JX2bVf?sTVsPjNzU{+qnRbU96@dotmb{Oar zc<}Stj7Iim@Oc4WnLn&#YGpc@@{^>MdL}^~)a!e?(8z*M$V&4u;iL526+x@2^k?YtiwoJir71NXo#a1CIAG>gHlLs_2J?4Qly|Z3;`S#pe0tYx(!TI79=#0)z7e&FFPFG|Tyv zG;E;@8}-l|oF5SliQjzv;7dG_n$~v~4FoiKcLb~##y*GI&~Ayf~N|QIheTp4N6R7z6;L(Ai*6)m5fv zb;Et#L%wk)I7YH;<&^*azYvD@VIuvid;~{?3;q>VSm~EZFb#yNj_vY^@{zZDXRqvI z&5t~T{KkZfHie${5tjKpygYnzUkTt)7-w;n9IlD$$xGnVMPGRNatn8V!y_1=59l${ zLxnyIsM(uv25NOgGett?$(Z!>tOEj{U#20){%pT0?6v0?zE{6!d;AVCs`(af&8A3| zntwy{s4N%wkFT9hpoS@!)8)rnwC0xrNK~5Tyn~P4hdku}{ebEPz}6NVJHtIRb;)SB zgYp^c)IG|@GmHRk>*xVH!S(>o@6drA82`Pz@B-&(F+$JJcxTehl?OsfGdHMY&|>hJehNla1_cdkFs!OE ztde(x_lIr{i%D2{Tt4s!+q1=QpUD{fwYSZN%XYwv&l#{Ik+38alYR@dpp;;piNb8c zn`l%+mo+=ADFQduA#rt%Js(~33zG2*F1K<~5KEx>%c$Sor%cvP|E2x$L#hH*T!>BO zF!HjT=-n)dG{8np76zA=Jm3YXCVDKh4yRs7_yS}f>gbEx(a46X%H6wP-Lr2e?MNGx zQ$+T<{wFH@R|9@yj$*Xu_^{|bS#FMC><(ppPF+xoVWqDi8NgMJ z$nBqI@$+LWpn%1_{^+Jv%r>R%VnDwO?J+{8at2`yqWnZFQ(M@>LuTQ&l(1p|zZlQF zKgS?cq1_LFzk-?wFa}|5&%&v{t&giEStrd>thC!H~v& z3D+XH{dc^u)Sfo>dImjT>-}$99tBl*$y~((5V7g3%XGV+y$t5gTF3o+ZYd>W{ZiG5cmII^i_j2HuUtzvbpPhS2N$%c3TmajaJmLpga1 zU#?vn^9st4;NZxsm6pNF2ZfxZ91mGb4uW0D4Zjg=ZuKGBH%Xk+|iE<(6E|5cHPl7MA8uq9bvb1`(y0bWh!9& zSrz9KNGITXigt=M{RxwJH=c!Vu7G|mjlrCL>b7(;;B;iaC&EjK-pcy7BgYw#VHh`S zLsv_y;c3-3P~~{&p~(76g&@bYh9P17t5fTC-w`0b`rg`(UQzi<3#hw+;b5+I{D_w< zna>?hqXhsU7%Iw0eqfTAejrvlv()Q_&QOn{D>FChwj&@u`*O)h?FuI5fMLDwz60l@ z9~z3RT^Nd!oyGrIB_5soM-6v74Z9CIRKBeruOl2)cgxHwU78? z!<%dYKXSA?m+Pn-b2_`mgP)n_Lx01o@KLlLuQ)Ss8*5#YEPOrr8NE=&w$%BuT(I4s z_wd6~pZE_&;bQ&ztu>a)#`DI2OFASGhP!3Z(`eJwK&!rqKYuVIzkvzDyfRDY^9~%h zLs&)xZ4RASx_bQ+N9I}FbqD!YsOr;pp`@5l&Nyda#!@*h`)7qtR;HeNYfYh$8QL(} zO}jaebR}^UyH}5apy{7ZV+01yECXyU4IFO0G^zF@&_xZx4ZWaYg%|OdfVbGi#na@}vb^csQE-NgpfhZ6; zgb8>NFI(vVZv{)hO{CN`XuKh1LZ39I{<6gx5-eWJGN)c&4ApRmpa>gMqe+5$XI?35=D&XTpZb;EOMGsR)wft1}Eh&{7LY;;WqC2ORvJVd-d=u z;8B1wzL|TU3a;LkhX~bnq1x1s%58*?Y`$@mujM)OfLcOHIowm{B{&y27YTRMh~q{l zTmPVt93$BEKJ{ew-6VOei6&IANH$m0^y%iYKx?rfKR~l@BzC(cUH_y>JG}l*diuwI z5sARs`ui``E!sNy)!^Z0A_P%)ZYFLNswA`8q^2%Gh}IR)n72AF9^`%0fJ7}etp%K( z{xdMgq+*mhwJ2z}CQ|`txrHQyKXYoyo!N|TEF0-I_`u%9Ugup zGDQE`oEN`d=j2DzLSeYITd$T10An{>f6Yi!M#fed59p*f46@y@N@7r$PS$xB+hL@L zgEM`k67GAvkZ4BT*3|Xuq#>J;CD*iCzug%B)cd2#>@Ld8tY%DL1MbfsLu&MT@$vZ3 z|HZ~|L55}8ACh9YK;M@7*g}2QgDU&fVIjaoE5oJzAoZbzO5+qlr=O0RSWF}xVnzi8 zFKzJ8^);g?mDc{NG3FBd8n`Rlzv%81=?;WXYJM?_$FKlDd|_?TuCuq}V$xr+{ukF= zT@VAheJK;vP$zxws3}t~r;PWtwsm$Fb+$zLgzyX3QTRg@iL!Pt8(*$$>!sF%;csa7 zzkx;$e^?L|aU*kBgA!zt-4sex^DS6-!Y(^b#|!UxVssg*IxYUy+dGWkoHf*0@rk7@ zoZ-3bC%@Q}O8h6N*he5SA4uQ!7XI$7;Qn0*KL7FS=|V=7Yk>=6! z^DV(+z&*>Sf2;HVQnx>ObL@&m06;&84+iSy@_O)rKo|f(0S5s1|L_GJ0Ri1aA9kpu R>YqC+%Bsm!N_`6YKLA9V(YgQt literal 128971 zcmeFZbyQYew?2%3grtOYryyO@rKEH>NVjx@Hz|#Dr*uenh)8!gNJ~hE^tbNkecp4< zH@*E$2=r4^8d{nfsWy@4^qm+aG4sSq6$+}VuRpRc zJnF41!+-kHVa9ZF@pa=-O|?I}OdBlwZk1zy?5rf!2`kRa@o4xb4 zoMatq@-{Cu(OCHDwG+(`J{FOiIUi1YYZ_Jy7)Gr3~xgYzxxrfY%6t~0D0p16YrjnV#joE1jq@~b{I*vAh1P` z>;(ye<)W$1wIX~}C;~yR5of;(I=BfC3$+>5^y+<>9rt*ZKdu(~;GamX`(NEhEyNB+WS4m~^~C&v8NKpGcvakG1eKN8nkX|6lDMB$GVYY+ zf#cJ;`AgN=WDiA=wgI*szjw%H%m!zZHf7^S)8be>2=2_bMsQCjIZm~NKJaW;iNO;S zF($T^XK=v8w&~Tov>M4=*#vs*rZFE;!&KxeraZ{?a>M`pr2>P0)V_;q(B)_V_pORK zN>h`3Qo7z)1P13%Rt^FugqaoIjFr^uC;SqG2MH#J9P5WaH7lH%0Nkd?NKL1AC4Brx z_DTdB7&Sx2xhJjxK0nDV5Nvw_k{w0?sM#`>Z#y z!*Wzr(WXV^#*m9p_5y-)s(v|hk_?NZjK6or)efD>^%|!=$MGa+gIs2_j8UK(hT&sR z_Li~X4Pk`;h^^v$_lfC;Ox3fp7mLw#-&d~*X9Ca+$s8z0zG*dbl?@vTpzr+j_#@^` zqaSqClXlGUhpQEj>_-Qd1-z|4O=l#gm=yIFY%^>bB-}u~K&rr+tkylh?k^QUW_1n*18#SwxPd=_c?bh#(R#hu`Ivvdy$xu*{ zJENqae5ANKX2FS_%EOeBsG#$+J&>ZeMn zIjTjfInGkMVG|##{i{Q(v)FY!avbYUm9}&S#2HX=`4glPh!YYRyi}P>1T<)hQ;QIa z9M!_r+=?>QeTKf_2crrjgjB^>#W^I77fPsdsG%qgsxqo97S0rN7gdx9DGBQPM0wWT zuw9VS_HY^nv(}`PrXZ&9q-6E%Mw}7!5;eSEdT`L_ZJWjax>nvfNk45d)tGZ49hXm# zos!?AUZZsec~N1S!}Zp>UOjR~s~mD-wXxjm)RPn?{Q9ASyTmq!q+C$JE5tSAGA61* zK1RM`gf?T653c#0p8fK><%{NCm)i?`&rDD6whK?X^9RG?8gI?{GMO2Ine6s@-^zAP zzZT!Zhe}AXXu9ZU(Y#JL@vxk5qtrFeF4rzmD#5&diL*$;h(kg~{1$w3e0xqI`@5AN zQ`DU%Dbh9lPAX`Vn3H0Y2$Q#FoE$7Q0yWw-``d4v@XlVHF>UL;C<@AZ8~nDe8&`H} zqCeKYsk?eH%y7$!rn022*;KCXvWmIVuJW+XXd!SBXP$GhHQkVt!sGh_@(!JMolEU- z{j#OksQfhZR7GFK^kH3FkYI27!b3CMFvXbBXYuf^($GX~4{U4N6<3qj@~*nubBBMr z`{$*9)vgw=&iC>>IygWHf^<#INuH~F9;6$MBkoC z7sR>8xypMU82#+LoI4!f=s$gTva~(Fk$=>ES#sgIY5QyVVrA=osCS8O(){bFtbvX} z&e8MLD_=Wblsoww;|I}){D+JuB~Kils6D}eLxY`!BZ5T?AP(4mnt@6SAM*4oT03qo z_IKoB3<6XabUjQ@+6clI*y^~K^h=bw_$zz_8N_@Z%bT~)d!GA$+Lpp3z-nSX6^*x4 z)t|Cg`J|FtpsL_OXp1K(eB+HuWo9;%ne0~8UR7#*)Sc2Dy0Ng4uwK|b+uNaVtU#~u zOFBKKk^IClRvk;Ym?kBjIIES>!@}y@6q?Fy0k1q`imOxe?m6!y%g9*jXTgoRcljv? zDIv;8FOgq*DGSw5&DR~|OvOwM7Y7gvk4TK5RPeY7He~H|GVTbl*WO%2Z~nevN?Mbo8o}KXeZE5OYzN&XD<&8jNA36sqJ&w%PZGVNIjY z;86ThLPj@+th2rf`}N~ojyCq36(S+%`t*^eQ_1Eu- z!2I%VzV7gFvPYbA6>;RYH=2&tDYw<-gC|>eTN#YR>NeUZb@4i$#Y6RFxd&%5BLlL+ zi}$+MXMH8LY8j;&>a%4O70=6>nr5tjA37$Sen0Kqr~PWJJ*sWc{O~(Cg@{@GUD;go z=W3W2}EE4z6%QLC`L1-uK9P zeRtoyAbnSW$o2Aj(>JHW(c5%kht3VL#kURNk}OTSR_zDQO~qYR8+}Jn9b~)Cdp6DM z-ud~1b)|J)%2y|)`Wgy)XC1DGo_!aCJd}3Ms}U_8w==amf(=a#S*!i+XM1*?%WGcY zN7#o+LW;hMCyKX5zlDb$$LBVtoXTc98a9-M^1PyXqYVU}3tlg2+{ItkuBWWj-0WPA zF5bZ1wmc-?NUnx2;dQ~$_?6r>UFXjacKLZcPO~Myp1YFyW3_oPz>{jfw<7PoD1^gx zeylgrvUZhE#wmR8@ay#Ct}swxG}zpa)~Dj`{X_nR_1x?&6)zQ4lMRKAC(buR@yQ7o zcRfTHCge<#W|-_TQ#Y1k^6J}M9LQs!=@m!u>nDmkXd+ARVZN^sRMM{9}x+RR5^_I^QZ_^djf*%YV6oHxNrRe^BD@1Z`|8g z&6q_ql2(1;h5Lk`Okvxu9W2)OfskG7`HM&$0cHg-;Y?t+wmjo<^Hp+9D!B>!uOv$Y_l zrkopm`qxeVeV;d`PR5RwADk`i?Z~0`HF|IF;w(r> z34PFi{rT5CP2Da3`$=|Ae=iFxkOlf17B*&9mjAjpxKsf8Q$9sYcT*e9H(45ywfMKA6i~V~97qj5oKmg4b}w@O!$`h?c`}7_!sj z#q`7UB8OW+p?Nc1$A!R^ynnnr;3#3P1UIjhtQ%q zhFHZv=e|b!KZsXTsoy?5ry2^S%hbGh&%q-gpUPRhmrwxnA7b&pqYF7NWZo=|$z$`G zv6+>7StUAkz(0j?aA|5fOmHu42Sk4f{cUN-ta=MxJ@kfhS?5R`qO zme#`lb2chG9M007>{9>sY=8D^*6u=`wR_vj)~fTQ+;(}x%Cmiy!_Fa&QJWWy8athLC%W(D*t&7ns@ReIzkM1!>UgyCy}!^n>%tIDW@=cPpzeQ+ zBDpQTy#k>+N9+^8145TsPm(Z(m4fDfts0c0Br? zUCbKLebUm3Cl&NeegcP}nwYGL`B8)xKE@!y=#8&hPoyg5C#rAmtWokO))9!|HI^Za z_@s+#%##vOQRzL>mg zGtIu;^5%S3g`tGQ!mf5)V#HnR)#;Rqf^DYXg9jBA>l0YaUsBmCj>X224_@0~Ej$W> z7x5gV?PnwWW<3PvoIbagTwt?0v`iD!mf(ZmTe>`C2cWa(NkoQ)$KEVJ4=&iUBt9~B z;Oz`lAG&ch8vA7u@|juG3gt6~>^ttwXtTUeW0A=`HlLrJ4zjd74E2)v*=(mf$0lc8MXO1x4AralKkM7`=2_*Oyu0Ku5L6`c4u*hFcO?--#pIZh z*-E_umbWSqh#QA|vlCb#u_u!6LjAQC{|P}hr{;;#_**;|M)jMkBYU$nTW%_<1XKgR zyOWUX^GSta2qLM4zR!gk*>X4U?Lke(i8AB9N$X@u^Vjy9ftS5UjT`7^KLap=PkJK= zjJcPcCy%|_1~C}lV}LoKm~#s3m(_fqOcgER8z|DG_dXluk?8qIWZycj`=Rrx#A@BZ zhlXWW4zLdv{c1s}ZGQI`vuSOY3xf{fi4G^-cxiRB`ilg3UzTJEeQ$Q@oW3K|CFe!) zF}%xFODTy)&8r#Vvv$vf11FIRA`vTFvTXge%?w@m?t{}#hFfZax_1Aorp;J+W{Zp3 z&0a||ACv85V|0%9KagrRqWHJV7^3SBlMH+v93@IKfkkD{j{F`M(>=CR#_6{sI7V1g zTeA~(uN@^h{1Vg|v1%Ppt9nT}l5`zIuMcWQzYFNQuLY%+e|(0r1#lV?a=Pn+rCFBG zrNb)MzW216hD*nNs?C@FXM3~#ibbIc+E3fS!61~wKNdx3H;xvZkt0|1>{jWJfA+dR zcfX=#gaE_G`vPs_26qpU)OF5CUf$9_@Fpp>kA}bz+>PKui+U2BcN25=aVUwuow%R0 zpVb5M2iRVhOfqiXJ7$QG-?dB&n3ew>jtcRGLW24|r^ihrc&6^P@Z(+0pqNcbQ|`oU zoB}Y{>nEFdr=@=;J8_Yw5=HVN5o`7o*htHMJ59OfYhBsG|0}|aNPT0dhPyS!*W7q- zh^Du8{T`7n(>*u*E9cFkQ3y%cTeQ(tr%^$O3-;c+PQI(VKH>$la3T9;xQjyKWP8WM z#n)@4Fy985rf(w)7?X?hcM!Ec+(@!ER?qWatQqFkxW?o3rX)H~bqpi(nMZ#&tO`{! zVhzYBm>|`1K=-p@$x+r&Ll<%87+*8MAY}1+Q6A8X*c82p?u8$Nchbkxw3`KMyhp zbGVW;lj+VA(nhI`?E}8IzwzdzpI7+3^oS#yvagO?{oY!J7xqp5LtnJ|amxki^8%H& zB+d7s{wM454?Dt-{ksiXa5lQm6{0Y(YP28kWYF=i6Vh?nC=`()5`-A)HNxHQm9d2Z zoZU!$7la^iKruq((QXn@2=VfUWwu?d@H2b5|Nf4`;=FrMc6FSN zzNhCaZ+d`-ww^2FzW;(-Smwa3#fm-IG*Rojz`+1SBq^z? z+}V!%%f)Xa4;bn?wpsp^Kt%{x~}D$3lZ6=v2J= zN5w&?+y={yG==`*e1hDAvR0OOFsj2=7C3FD`Bd!ir3>Lp@?1JKfxfa=b7_ zS1I(lwy)UvL_*$;vgR5jFCzR7n|2UVKVCfC9^thfDlDiaLG+#w%os%rULGgC7yd01 z$QrrV23FqF!{R#LJdAPRyUfhe^1elrb3_~lmW%x39RmjYX{mtzk6;mActTSF&EF64 z$*iQWcKXTfh(XQd;M|A+O}J>t>5c<0AuiTisCLq+2E}VijK^Q*<}C{|9K(v;u6Yxx z3?>Lk6ILr)4pZ|`C&Rj^MP)emA70Vf#;O$soz_=O`tKHI37z;b#Ut*Jk5#}EzGW`2 zo6&|Wv#?cGV8QK=@LsxV&JOTaZ?v9`ghw8+m~oBO-bOZ_KAuGT{VBT+gn{1^G0bq8 zK`!&U_(7v#<$fM4yw6-GuC;eojFCGXHRIxRJR!^C@(Ek83S`KgK9-iRF6WM&#s=+K zoJwHADbd8d6A52&x4|LH_63P=t47i{VKf2W3Y+PjG6L{naAsfqudRP8LYJ1n`8S%$C_N^YS!Gto{4=k6Kk~ zacA|tZ1FWLxmVmbavk#UstGL^6bLZ5=L`cZE{UPLMVpN~n(7_TscPk}`cN|}i=b8`Ng5_5qm{qQP!wh7HAunrR$m}@m5`Av=M zA*m-2YIs5;0nPA-cuiK)cc*VK>5#!@OVd9`1m{JB=y{B68p!F~4&jsdq>_<@xTyH7 zZmJK2iBb(*-(4-Sdn()-PH;TzqBNO;JaTD9sk`rDeQCrNG$XVk?RH;Q)orTZq zU>Nv`=gDp>9s#n}B#o>;&pFrb!wNf93GnM76|9#z9CVAWAqR3*}z%9g~r>Bd?&u?y0W zY`pFvZryif2_#r}azz4&#An$BV)Q|zf=nDTYgz>l6KH2DOq=Q&*0k73WXId8nSyo%ZHZp_A2 zR=xqL>(F5wdhp}P$LJlUOA8fVuLb@DDGz-G#Opb2>j_a+vp!p=Sj9sT6>P?K5c2)C zIvyWxE6`&h(BpLpPb-(YxzhRNa9OR3Gq@L9L7*xC#bIs-WR0t*7=q$_1yd1{pGQM5$(ng1dEAl5zX41{6@&sSNt`;?XgGJfI9|Ol# zC>|VEGT%Q&QLu77Fy)u8==%Ctr&U$9Hl6m03wz9}E@pHPYlgXXvLP=)>L^MV$?MOj zW|{{eSIi?vZB%)H4tWcmlO`mrTtBvS$7NQJ=g+Ua=vW*%Vt~MnVN=UuR!14s-T;%M z26hFbF{-`k-(P^6P)Pp;u(Q&JA!nWpYILkZ>W~tB>*a!VN^QR4N;;^}WUxhiyE(*s zyU*4b;HzoJA~rf@ZdBHL+*$Vv3w_SIj_2C1G7~`*!0ZbYA?z}Kfm`Hyt%Xm@(sn+f z2%&we9Zf3eZ{T~Kf3QvFb3V>{-FL58rG^UPQb)MgR)R*N9y&6BsLt>uOi41+@?M2< z!uRJnt#bfP$j~5vP2wLD*el8lBP8v5#|y0oga3L+q&GNpV)OYzleikXpXnFD+!?>N zIx2M;=Zb~GKSAQed9tY?zKqTQ2dSr3JUTJVzkj*R*%M%U|I2K_9W^>-otguRXnsp( zuaJ~jC1nMzISg~nq9$0xDF}IF`;0Omv#XB)kYuE<+-KW*(nC@Wb&KU^A!_Pc`CVM{ z;M!?AMcXFTlBZOXQK(yiOYDBYsqZAK4$Rw6O`$LS%`NIFZ3zL%gGS!0Wi7)?L@8@EB`yGN+?MKquC$FWquPNL^^D(rK{6C*80^nk64If!`}` z6yA&x|1LDL;&qfU_ly1qnCwy8mA-~Ln?|Xe>tYczJh>q_?Cz*w(y(!s@6JZ;9yNY5R!D0F zAY!W}}zB`cM`zrEtYrjtiKdwo#Z6|oFAQ3gQCHnb=QCo8t9XXKa<5oE;QNUA!_*NIZ~ z5d<7PKKB4IN2rygRp_-4mP+w}bI##9_A8I}p?1LC7%kk$+xiD*2A|8b4rp1`b4Wqp zI}70wNCk0CaEI6c?C3UiB_L^1Kw9G~yOZNtHkr|LTR!%Bfv7Bc{(EtD&Zk7r)ipNo zwidg1=w)I0`(ShutAgbBVbx)tYe9GsE=(i^yA%1c@{e+-C!p%(X@5ngbm|CD=$Qc& zD5WODoR)^#ld6S?gg-9=aF`I-6`?19>5_G(iWi@BA1N#MuR;_SqU~I?{#vc&&bup5 zPtK&-?MS~?W~RHi>XtS2DW2}weR=EYfP9lWr)Ut~h|sM~4PX~Siq$G2(hy0UW12wq zJsd+J{@wat;Td{Ri@1dEjsrZ_L+OS{+?Lj8C8c)TS3LKdLdnPbv4bWgMz=6HX;fOD zhy*O!w_o>opN~s!cjFmw0`Je14O?TAi7`gvqrzs^#g8pi1iq+%Sy~Iw*P?T(>OhBp@wFycrAxq9su_s-|pFV-0APC%Ra#f5F z29ijj7=2>+0+;~?nBfe6rh`HqA_6@>02!&i%T{Nm4pXaK*xG5){ykZ!P*pN@en;ka z_xu$=G9z-yzvTk6(rl`w<@QC5fHz3OPAAHe$PNiQfyjz5qxoyopNkJcJtvEZVp|7m zmQUtk|7!<=B(XyvBb-Jln@AHpgqBK?Q6ng*vJZDJ)9aAn(o$X1DCj4c?@OGzt|saV zb)Q_8_CJ)G<{2)!yp7rys(%7S&OlD!dKCQf6w?A1_JSCrU{@hjc%xI@ zqhR)58cAv@G-APO{5~a`J}#;UfqgQ^5gd4cJ7}S_4fAv0a(q}t^|~?>wSIRSWI5iI zjKTh3D_Nwgf56FQX?6JC&uxr%{ltc463NaC^NJpAuEW?$(rd!+Q`BULH+EOm2T3z$ z5gV_|_+_0}7FMtq$9iN>)qkI!UAfm*sCW4c=Ux=eo@~Y` z*wxy9_PA^+suX4 zAHhd*zi7$+4!|V|2y2gAO{?n+msJlin1NF~_^))r3lc!;S-X-oQY|G!0v|(tB5R_} zP+%@h!Ng}FaIH#c=_M`{t`A}sxrLN0Y#F=Tq(~+Ni)5QY^H41JkC8i`SzRYNd_5Vj zmq#07O#qMDHFXPIOsN2H85KI1XbrB8!?b|!Zc`lQ+b`P&u>_--#QV6@Y_sq4+SOeE z6s+h#VHs*6-BZ?RH#GQF51 zxSf9~a=0GM9eX{9(ggS33H$*_kdp_50H;-9D-4J_rDEiXA=8aYGhzgy&aQNslaAH) zRD%Cs5?VG5^3OO2w9CM6Iu!-ZOn9`K{rZd#_Nk9ID8P3#3a!fQ=jAetJS1UcK73-tdSGUt;gT1 zEE(oE*to@+0YaCPgt?w}Zisj$o^6V3B1l@M<|o4wcw#sxl*nbDd?`l`byg6d7-(Z$ zLQ0mxR9n1JqqkO`uLE0X4dy#+^lZ`p%nxkvicNrT1IT;lOBx_pWD1P3wXmK)Cm)-D zuU20o`|#}vucOrI;32Y3>>B&vtGGD3KgVlTmaGCj>uZ$o3;tj)nd~?s14UFY`#c3S zOCREYvXk0`NtV522wkgsX@{KvmPS6-2w#1un6Nlul@z^JwZt#x4bZN2)A=_zj7>0$ z##yT#z>0WwU-&_zWHB`=rz{pFEWCD_$V}KRlsAkQ)qf#)J|<2>VU-&g{dIIaPqMe5 z^B6xoBplNM1-9>Ned91E7Ob2o`TmJv`HhNyICwZ|RY4JTaBo4QoGKR7&o{98n#I-Z zWJ{qw<#>d20>Vex)&xt(9hUR&1uoU^qr5xm5EGT+u@1jS)<~SI5{=?-A~iS^72?@- z$R_2DNuSxQM09;FrUO>}9tB^#_gyVjDU|O7kQOTqq09IcDYIBM6CiOH*10F8kT{=0 z%6bFp_7sqrH#!U4f28Kd0&!rSTS*!UX(MHSmeq_5DAE50MIl$YP2FG?tC5)eZEPb$ z84b+KjaD&`-nL6k$K#(qLBgR>L~G)7{xxQJvs;K!0jl+%g;~A~%BPR=!ViBoa%^HE zKsvycVYp53Y~C#xl)eHA!5xcA6bRgjbd}>0L|c6yK~51;O9PNXgRbR;f%nNz!1`c3 z0OS>WeSenqI8+?4B$4!N;cDh8mnn29?~IKyd?!x6S_! z@L00Sk(PM|7XYxz|Go$C-M4ns!yiS$F^hWKD?xS#QRozgTZ)6Pm{h&^-|77aRwJmi zBDoq8Z4M4x?w3cZ zS?&CgyUADIiY60YX}wzUswj7@%WJyXD;);3cD%|kD4d5hw>nY)c^%31hOiWe1BIpa zgM7M`x5q(M?+{r3-rnW)PF5O}X>J9DGVfcasvlTpfXQOx80L(tsvsD49Aq+op>2_q z{@NQr@oEZ1MgYaF^^j8`aEPhC_ctk%^8CYqg863fOt^e~m>7`MJr($lJ7rZ9x{m!c zW`Lze+xQb;p6Ss;ufT8A=?zMxo(daB_8x=TLba7NP}Gh>rI!Lwgvb}j*6X;1j$1$} zCAl{u_q_cq7prB9D#mEe+rTQEfOsjaVwQ8#evcMhpCZCFko){30u>J&X3)|XzfCk;#%L(68D9Q8zst)E={h}Sm z3{B}i_I#-3c1dXk`ySETE|AVGWj0&cjIoA3;l&0LIOsb_s7qla$Mq0ZjOoHtOL9U);|k9YOBIq<3u92nTfS!dMjzc48PM@o z{>lv}Y*|&*LTk{c9s)|E9Vs}MJPz;sVkqpC18z70ip;ILrQqGjq2-kt(Ym@f66Wa0 zhAJ|xK)R~B=UOF8I8bhzHs}BO5pF0agfumaWEBKbT8fW%V!mAcc`{s!fg*xE0a}-Z z_!#y50osLT@7>F=-NT@i8L;_Zc}XN6e#PfX&uKS5DRtJsZyA*CM0WZ0>X?=^(WtO9 zzqv|18G>Z_K=>(vH*s$T4w!KeyMy2{J_i4crK48HA&Pgq$cm(o7#i1UpdzFq6&fgJ z(O~=XtQ2WRUjco0D3;a(aQuJq_y2zG|CqwxAG774Qt%{^LJbsj{GBxaYo7n_@i8=j zbTA}x_M`mM^I>3ZWx&w175CJCZrgzTU#I=}x6{_VJ-hj{5j_lI zt@$~x!}=QzAYrHJw|S<3aGeSwWBpqlxtf1;?7zu+TO24H;sn%j(_e`fG-OynGZ9oc zhmu`qKv_yn()a9l-ka(#t6#7(o0Q|>BC@H3K$|KSK4Hp%&_x*ZO&mke#IEDs^=vE2 zAQi~y40o#cKw_VkD5kxD`r#Pln`x4s8E84URSJ!#wDrOX&>?9JE1rX(r$k^IG(d0x z9bX~po;P{l~3(xQ|nkwO(MFwtLTQ47t~-ax(|S4kmL zNagH@B86>`B)H)aP+0+45|QG$SIof@f(in@E^Ikv2Q>dWT`jw%t@_?F6+(z?Yd<#u zu;esidjIq~flKK$K!hC@O6c%{(sr1kwCp`pq%a93WK|d3nnfh@HbfUV7C9T2Bug>s z3Qo~;ozH)QtVDyP8uAT@S{u&XP?{(OAYo{x#0i=5FhE}2a!$t+ z0i83s2`EqL0u%TaAfq&-|;}F)^rZ zvb+wP?5ZY!3T#PDZ{u#!@LPKr{Iqst>S0Mui^*DhHUa4xu#mMfp34{Ke_0PC%W$7@)FtAEMt` zpNFbKMjo*cP`7HF^yvbK$4)U8cDfG?g08OX(5;xM*=0;yw$3i<+daNU2wx%&tu)bY zF7|g;E`VNC%ojBEde*b=D^#S3PulZacHfBPvg7I&%@%^MXx=ON=9%L-+7ZQw2BzRt z33weu*Y75Vf;vy>vb?Lkq5*Mt@Xb>Xsm!hP8zC~>MbiMJRU^`q_9(Kt0b?_${ii3DVVX||bZ|udlmEUgkWV5)RREmlVELmJ+7%7xYkb5)(-+^FR7OR`~pq8md$ zl?~7rHY^KTdwz}kIOKo}tA$yqPvZ|%k=2($2xx>jQAy_&fIsvb&HWVImzQ`tSTXj@ zG(Rb&obzO`SsvwJh8E4ip!m%_vL=h_Uv=UfE)7JlGxqF27Ewid?5u9daj>ZsP#GOR zfg>EmL@qn_5+HkB9uUkWaFJBt@tr`KK>owVyg|30-yl!vzLYXSduLd;;#ClKKyOiw zP3k(a@yLN4Hu$2c&R=MCZdvk@=iH008%L!v6mv$!08wDo`z$ieBB%X_>!K~*9Vq)q z);R}}g^T5Ei|g0e{7y>-a0baY653jZ-GlYQHLK?* z&@xJk*k?=?=bq1$HHBi+smEi4f(r8hNdUC7bS^oKk_@qSp|0H)L$=3q#Ji5lP28s< zSo+!Tr@tRJ0gLYK z8N&UAJlj{~tA5+kS+=>ZG4H*dbx^L~B_ojVK2#2|4>k(a1Fg_sLeju?UuLvBE{C=M zhhvLNNO;Fjf0Q1E9Ih&*I72Q6!O$cogDi;~lcAL5Wy7Hql0mC2)nOsIy3Jshi-XN^ z&dGP!{3+T99Q*nU-E}&lYA8n2XBTnO=eLTsU3oXU4{lOfdvSF#l{&AVg7VF?l(Udx z0WgsicH;L8C!qbaC4Wob0@|&Ft04iGuib)N+E`5;D3XQ=jwO%3I6Ci-rqR!`@GWj3 z)=kTCz+r1+*OWN6TLnsk)|9YcK}qb{`3ZAIxyE0M4J-g+ea4Pl7;=p>NJZM-(qKZ3 z25H9Uv{-`?6Cd<`WJ{+I>O_(W3ym=dK5b$?Kk{HCAhl-d(D%D{;R%&z{1yeuT6K35 zU7{(4*)uRtQ#;FUGy*|SXL6cG^@BMdEVy!S)fj6L7I6kPflQYfE#@}VWa)@dR}5)9 zs6Y{II|z6`vN<1OJzbBvm$=lAQ|NMI7Ck4xC6(P?U%T|A_iIfvn$_KKPR9~0JjhnR zWtS*zmVMf<+&xXrW-B@q0k{kG54MW@CARK3WEX8yh?D*Z7ya5aKvO}-C*^2Q0bdXn z-p%*t_k4T;C${#5z-3P)9~me*3dvdUnRm|Ezzut-05(23l%|^Qvr|4)x9ls&egYdQ zk*&6+QI2~R>1t3@f^o{o7`v&02B2Q4Xr0UH)E-M@4x2#zh~RKD4d@eK=e)V6;F&jx zd!shFhanhu6V5>o3bIq?s7u#Zo6TGutdC`ktJJ}eyYq_&ed{jcwnd%t)7=`ia2wlG zxy8cb#HF#7YJIylP~X!S^(hu7e3r>8?@DExS~#AIh=`riVyx6UNe)y!eT`2VS7X(b z4CPnpBWMika@=1Sh0-Ja1;0E^TAIr?3SpMo;Oi;OmL{8dmU7T?E8Vx7HrPA9ouv44hqU3 z&(ts2Kk6KN7>{{CTZq3@omGGDJn%Y{o_^t%8zw*l*5&hg0O9CbiSUNsc*MpI!>8Wn z^$cSY=U?%=KlX6BC8V0>C6PzkiQ@s#UEZaZ8Lb<+zw=`RNrGc%1?Z`siQ~kq0uq+B z<5hf~jMKjFi{Wg!VcX8MG*oK-ATiIFN?62~J4tQKRRNsgrDw zYfJ9L9I}R;{O<;WshU(nLj6W7^>s)t*mMy|05j;aq+M6Nj$=O;eH z-3P>qJ>?tV$i~S9vPsOH8U{Yo6w9Co`ul4~whT@S)6cU6%OLtm$8o(8=t-YnLb$P+ z|4^pEZ26Z{#?t6lfL<(CHp24O%U#t5GZcK%$yGM{K~R)rnsOqdMk^88I(rW?H3W*O zeRlyG+DO?25crhkS3tgSYNQi23yOITRuuNH%_{JGrThLj6$gmYe7sR$vI>rUnIfeF zEI3!YS-6`Tp23@nda#!9&eP>u%}bvRcK^J*03eh!3OHlI&U^_kXwjLXcKWJ1YtGB) z+3d3aqM&WJAdN3Ofw_Oj2lT5tH|ih!07MFI!6LxO;cu<`&LVgxS6|%N>=MlIhx}`| zj(A1bi{}`Ai!I6X?S4@tWdh+ZD3|k`7d3jfI zVw6GNmx@{i6}hIhtH}8DMTcGz?7U_Gk;Ofxnw)k-Zlk$XKvT>0a;$&>xKYR|3&8h| zu&&V{wsieW{b8BT%hq;E9UNb8HAuWY45&jdbZpj4c7U$6LNN-ZJ|l>DMh<|s5)~a?2~c%vdHuqGADVDs zrl(a6fPcZOydOoFK-=Qo>CrIg|FzEaxzwWTpoXLFc(|GEAiOXyBdYpgaoB!q$6?Zk ziDyu)aLoX%X105X>^W!h0^_D#-&T)%2?8O;7;(0zdC=`7_F+)WS1Sktt=qbU7J!ZH z4%%0sxYNNA^gP$IL0^iPV6de8vd^<{-#e_LAc$L?4$%3q7z&dKs9q4EMbe9*(N)Av z4N-w~8&+7b$C*ulN7XAoc%$gl`4n#|Klq`|lJ?p(;iJT<>_)G}s_ZI__3t+eZ(fym zcI&#%oAxvtb3paw>%*3#A;8-2^(4Cb@znqcp61Z43s%H9#H zW)UvRj13RGq{&{L0YHfl#xn}^#16h;J+LCB%80-qM1l78KUg2^fPA*pQYb~3v<&EC zBkxF~TpyAPQ+3dXe(br_3HUXugL(nxFg-QPcA(NU{wn^;woPoTZ97!F8U!HKr~pWU zcBr?Yi6j|FtetZ~m=#$z8WYWI^_w?ld1O7BT@RqqGZhlm#0^F2jaxb&ePLmwgElDHz0xisqb1Rt zxI1{rtRQm*afM*FY>F3zQN5XN_tyaO-mG+1(JJl5{txBr;tuLXe;bl+%@Srr zQupktIH&KFNji4C#&mokt9gi?wCle7(_a(I#rb+cfS3{t4`W&WcvBoJ`u^*N9QQnR zs>?p<;IWhi3#|HaV50Y}Ny`SbZ$bsB9xh!)A}4>Z{oqdFT5TpGAOM|@J{h!r!&&1X zan_7H7{b?25{(qq3^Y}n?U^o!um_05VOs>^il@G<2m0-6tsJelfFd^|zy3lH1Ly)v zLw+QPSz13&$M0ksO6K`B)3D!v8aXjQL0j2J84jN+&4@?n$Yo2X1FwC}l$5KkMLLHr zmV4*(1-wHMxb^4s+%n-%NampMciB@=rAG~Rxq?ba3gbZJWhF`eH}p){E#YC#N%(p= z+_dP9BDpEske6e=Z8P-0HVZeKe-k4tbiHN7E6-uAB+2;Zn(M=UWy#+P;c9(hqfl!* zzVUE>yR+Gehcp!{Le zJ&~rkS+?K@YNCUW46i|X0o(s>>+5~90qB>fo)2-D7+`6dG60md?sZoP1_n5VDc=}S z^cJVxCP@`zgQWLO={~``D*if+i-=A4J?=yd(9rjw853ff$;O}S%-hK5Amcq{Y%M3| zM25&{Aezj#nk5(n^;EXoSA*nCBW6NZi=fR`&>q{9Xy1AwVHo)ul9Bwvrg1G;9$nhE zG%Jz%%Vz^eM02{CCUzfw8J|PY>MLFHHvI*y;%=B^B=+5zX+pN?b3y_456Rqqrn&0- zH{KVPbcoa=w?MD%mo9bC<$^UHnIU#skQNZl67ossmn+9=K1?Mt6#13*B#H>vu!_y?#n5jODe;hIk6EeT<6MvOuf4_|f|b z^IF+&(^f4@yHfnWOA^spG-eKUFG0bWKpeHwX|xmUFS< z!XLuG!c?}{-AXbN{hk7-<#^DE7v^=(Dr?02w2lw4+h1>MV_hnknhkxMh;+I*ZeNjr z2<7R^`WCmN!_81VqhIkmLF-#tDt!Gc1VrA5((Ero*OcHKU8r#%KT$;?9iV=_JAiw6 z*)8FK{`%)f^w~T*p+7(2=s0j^N21(F%73}|z2ccm{Xr2Fg!)vd@8?U?%*V4Zo~^5B zzdI^{qp$mu6)_k6*MTC#$WuDL%OASX?7sW2Y`mrk*39_I`I~cOA#i@Jr(|w@<85H~bD1<#*NkLXJ0@%2L;tQtgtrdJlhuAFma36B=g_ z;2F3CZgvbtlWB<)u{obF38xWuybXvien=G>9A_i}FZS?S=7lX-5LiYLIMr__pMfsm z4?czjQycyfalY?JX3e7q1FXA||K+kVPZ$}Z2j;mYnlmi4$~W(Bc#H66^=RPnl?i5DCgJiMBc}MU77f?w;_yO8R)OFK{{VIB58#ca zAeMN)ssr|)@6tRsKTEpOi?g_bUTr6361bP`8|$(~9vgWJ+0(M1OBlSW2%0u&?C(tA z%0Vm3RMiPwFaU^@+@4d%MNz4I6!{1QGj0CibXvJ;5QpkQ#Wg}hFD>nX<3A((M}eg% zvbL_ifK?5;mT6L>FN)i^GO4ZL^>2LAVpePU_RAih&0M9}e3O%s$K9ykLyd(1vq-i+ z{NBrsy9YAm1+A+Rn&%e5#VfpMuAFn9P?Wmz%yMK%-$vR5O_V%ttXh{>yyA(c>ZrAk zOLEbkCctZrRpyWmuj_1vd!}*QSEM=gp`@Q600HgUU=%vx+5X470lM#35>$%CvCQGF zwW)Vu452avq;;cln@Kmp4)Sg4v&`&9k|{4XxF9H>w z_Loh+5T@w2<5gn!1;$qUoxk_>Bt?CvSm=~f^YDjbUGG;;RZ@pw$=zSxi(C#G*fQZy z<(SV_$W1ipUmPS-RUC#9HL$BbY@zOnK9^?8V?p7qLAp*syNh;IzB{o12vi=4h9jZJ48VuZJS9Z z(FPQ4;H3+i?-a(Vy4d^6lLZ$J6yPyNDy1oL@J$Iy53!7{Un8iczrU8=O45DqY1+D^ zEnV>AY0Ih&z5M~HJV~xh`!0{j#mq-G`FZ-M9Z7mOwO*OdeWO|biZBe&2xGZ88FtFZ zM?g7pT!j3>*I#1k2ji$FHe>5hm6y=<3#ax{)=NzjfU{+*hxQKR(c$Fe_(~!H4cAo@>z#ZmpYM?)mB3$mtc61I@y-+ z|HIf@2Xz^CU89Pqq$r4VcS%blB_Z7n(kTik-3m%~N{2KEf&x;C(jlFKG=g;Z*|*R8 z{&CKn`JQja8Fl>O@4m0t*S^+XYi*pA_N*hI-*x0(8vSP_aM3bH(wfEG=qR(+8|Lhy zB#YfiROKk#&`bXsN8drnZtJEXNW^Pfz{ab@_C4FU$%pDAzqjv@=xsj#p~P8j^;I(V z1#Fr0pJxpN2@H;3pZU(sckL=?MYV2OA1=PwcL(AYhDDmQ(hYsZ?jKu8uL9Xir74Zu z*RIFuH?h7tINE0abQ$K1J+F<*SY~iahwT#-G%${x6H8A+jyjPp)^PWbE5bebIUpQ# zjX3=gnQ?!ANHorJ{vj996JQ+mFM6k?eQz!5F0=OO>CuB9MC6N3Kil*@{^Vg=49^-2 zMK&Igy}j|${>+|($b3wKOHN#h?DtSM_;>ulxfmQth#f;hf#&nwq#nb$xyRw99dp-VSDqnW#-8 z0kXAKA6r}+ER*11#^Z?)@ajFjN{(%mys>+~&5h7IG429{o^LJFR+C_JYj42j<`$%4 z>%gG-bCbtXsqWCN=w^8juJ*q)CJJl2QZ^%oQ%HKlgo1>JszqkL>v=h-EV-MnYUXVHC03Orw|RS#LGhYEl?jEKSW`sY^B;V=kOBfN#7njd!&<))VspF1+ z6XZpcs!YHuY-_4HkSShh99(!})0dk|6?``ExUgVWd0zoH8;>Eoll=T%JK>+jQ7E+- za(w}*(f4(W;^9rlkFTSR=~QvEdQq7*2l5JCiaX1J!oGeyZxOzr!+HF!Z&W!osyiac zwv!!Tr2$d`2>G*)`@`BeGl5Rnk;D&=4rMV$tlJOYYevj&f5%hC+SR<6Fk=gChRsS& z_q__gEtPoY)Ti(TCI|^wj>q0ga*kfo;o)U5`ag!MRt!l$XJY!L6INu((E4JkXcv(( z{`I{}g8EUGMg53VZMw)#2S=8B+3-v@Nj$P`3zMbsZgQt`oev>1GYas!C^Cz=opyzu z%#eo^KtvNCN27W#Vj|IMg4{4~$LXJtrWBDXZmE#aCBE>Tx2FhUzxFIuRLN0~33wEF zUy?tt&9_D37;CU!>Aa&L7ZTlP3)9N$7K#~9OwOQ?6a`6FG2Q%X3olxgpuY#>#-gzZ zGXX}e;@kN$d@<2;!0a%j70ulK36RftNm%~;rVXAmD!vn+U+z0qlyBXVy`$GS@BOp8 zs9>&o3jbu%k!2)SJ19AVFqd$ow^!lfKpli>mGc4{J1p9 z)4K^3KyJhDZu;3lyTii#M&pwIb3s4XEup_3J}8YEx*EDM`93j@(Er2{OMSDwocam> zFOGjxX#@BcOl4aEY3Izl`w@i(Q=YYCFSm^qj{|2pHzJ<*Acz*@rpiU~2x^g&gvNGS z`3l^?om;%(eFZv)yyhhZ$gSp9`;tEHx20aYd8^xfV__WFyj3V_lh-+JX(!~{pnagAa;tkvG%A0YQ#d|fb_T*GOU7u1GGp-SGZ>e#3174*ts zMfP84q%K!lU!-WPiFn#P9JPL{^g5bkt3(2XVo<|PZ&5kG`NdZ%9N&}u>l#!Iyij(+ zU*g`^(~Pxvi$SnRu{@%@s0GK@j^Qb#I~6&@+mH4X?&6u_=tGi@O#ny-Z5B=I5}|jY zcd<8lB{h!OW`ZyaAgtHN#XfREhH`fgkIRI3TC~g5<U#5?lpb{!oYnsDUdEN6yO518bt|>!JFdu6>LKsFH>TUr%Lv!>y_yQJwY_X3EvN@JrmpzlaAmj(6yM8X(T=+S z4t8(HmjU)Lfy*^ZO$Ggz-N#TY`|<<(qBhjWb~xJSaE~<346S_6-ER3s{DDmo9pBkS z#)Uhh)o^yb}(6|Eq zqEWQJ57yGkvk^eqqH8D~0IB+m$4jVB>Y4v6)-M0h=#-P9DQo67r1Q|1OKMM7?NZ5X z2p(A=3OnZ)N>H22m-)+?>4L6@4Yxrg_|lS=M@ak|)k(Gu_jnC*Xt+xsvaOAOv@+l{ z0G=iA&_(VZ_Q(HB+1GT?c{!{pSv!y;6B*C}`piwZMRWLel>45q8DH;ef0!)g=nHxH z;zDT$QA$?)o^qN8(-*I|8w7=Q^y4$%^4a!-=Nf?&JNiycWFgV@Oa7r^Wx>?#v_H%Q zxq-0Q%wDJz2{r`L_h3Gt*RD6wE96fe_eBbdY zPeLb6-9z(>g}$@lz{S6VNlktF0{Z6Phm$#%FR>y^g_q(;F3+yxG&T{RPl1a8o^pGr zXIafi^gl4H1G^SIeEQVsz@~eKqLOspP=@PL2(HV^3qwqT^e4t8b26YB>y~5ZgBODx zcZOc+VH^{r7Ssmy2KM}ofnN;kzQ=CBO;Oe%bI*J`J(KIn97Ob;6jX!E`tJM#N#o^1 zVI$lg^+$7|H=i}^Ybhw|ZQ=&CjeN-u){868TpcYp&VC_B-C?1%_hVpNp1tAjJ05`3 z1wT^yK9I7{GI%XUf>-cnPSlov$R~U;sNh|OX;2qWkWA0>$j-7AjI^3Mdey1rFSr9f z@-_ieK~ifei=qvT8-r6s10w-2J=%%)4Zpc6-8bsQR^7z)U@tLVnR+;erHXon?umXq1@%`LK&8#(g&|DxUg zx=kn+90W@&z1OLbRR!#F%t?CsLr;3h5Y~x zaUw~z;yf|1_5IAsP)Fz8sxM;n50ngFF;`ZsktN>IrUw-LTrRe?wJWhfuFYC>&`U&( zv&AAWNG3jKJIU9$lV~}COL*x^Ia@Vl$LHFyd@YVskJVk!P~N{+{dd;}JF>Bvz}TyQ zooPH@IU}`BdW|h{al*6G5ZN-dgYN4?U75)a{ckQ2hH$9n_~^<~Ri|+=i+muA>iX#H zB%&!!WDZ%o?dM0Qg7c%nfaS!IeDoI)oA}m(Lk7e8dw&dK0b<^hn zYtnUo(aAl-kJIgGS9gbCh!mqTAEKbHibI-VcQ9^+Gc!vi85B4C7ygAtq4=1t=zN2CpVb`8)+;gvogx;Tcf>@fHnP)m~iBTTL0pxEcTEj9EHuH(Vl9jm|LT zTw^l)UHLhyD@V~56@EEtR7479@a)>*L^aqXh2?{FyEuykVVcU z-bLw|MsG=727*Dk77zgvypIh#Lqx0e;o*rHn*e|jlRUgMeNk>ZT%4r+4m9o)>5|7M zO^5)z3FA>yBA0L{U1gtk1_+v@`s-s1+Kt6^?N4K9-51;BOx-jti<62c&(8J?9qg;U zW3?=$Gvq6DfXswJXVc+KNUp#0iQX>G!1z(1*u?nFMh!Nyh^2StyHoHzDTM^QnYYM= z$FyI8HwI-86620pN_~%J-y{*c)YFcN8hx^EYLkFE;Wm&$uy?fu^sXk;(!!S6B+5Le z9YpPE-eui$hkD-oj{6KxaethHo{~)Jm(hZGGXa`LGO~kSM2)%llJzn4$ zW?(~|_|L@o>>?$)@AdnV)1~^Kx4aYQ@>Nvo3ltcw=)xaR9NJg`OeuCcOpgwmOOJ|Y zx10OTvE7nhN~Tic4dup0C`@^#{B;`0uIc;wHSq?m3eGV;Sy-pYV}V&HmtoFmHhlbH za0rpP3>bbUL=$*4HnP-BAf~J3csA^NcC;<`k)4n|qYX6K+8jo7FtflZR%AoZRUOdO z>(h!iMpCR9%v$4jAR)*jG_3oVz!y3wPSV0fhf%Em+z2!Y2U6V&_+62FdxhTnJFgDP z4jd734>ED=ADU521l@Z&nK_p1*EyGAp^#bA*$p^eK#_-7)CvlO_F9od$2utgsY%1} zd~5R%mTlURs7-uHq8Zc}edb=|QFmS;=WRslCwVao29XilJBEfbG?2J;Z;o28Rpn;I z&8i+tW!!P+kSEC&zSv1@%I&$YfH-alN{aAtaWMR&O&q1Bfggul439S{R-~dY<{+<_ zyiddP;e3PVx#$RIFe$@wis9U(d}xc1_pWtcO-VvazKNI~8qYQV0&05s(FcxC6Svul zH2N`Ga;XIUhQxA`LW3cO9)CRCd0yUd@K~>!67u%sM^?qy#5|0S7a?vCBx{M@^-BSr z$xLSk;fQ$@E>vvzavqlSd(oL;20{^ljZvmTg%PJ$JW~y2Kgt-2oR(nJkeljFl>OMZ zquI3VF$=FSa}yN9@5kNb1L~h?mE5BC3*UUF!@-}dKMNdHW2dTi+Pfx24#sF4z42A= z`z_SvtBCpFXMd}eroOG&+^%92`t#qZrn)hejaS|I3s1kSBB!_#_9Rf|2s$4zp#Xw? z977z2k+*EHiw8*KP=W)tyAw|p~h*b zH&DusRhch(lXiVsa`ha$-P_1qYnIoc3MaQ0CiP1M0!%~jz-=ZX|Nj)}-7UAx9;}}x zn^T49k?>j@0-vWpZ6t>mI$cJH^o=wjOwA{0dkrj`z2xefwN!oF%eYMQ` zBrI`EIwm)z(8jnWkZ`lRH4H16y4hC?=~;U6xP{bZCWBo~z~e)Yu*L$9V1$_Xy*ikf zJoeVN#rLZF`w}Hn$uz+%GT_ZYiQM1zNz=H71nUARt-t@>BOQT>K*;fHP1Q*3i-~xd6d=+qD|Uc?{{u z4VugjroaHptE)uRVwvQqt8`7-_hoox<%!S4R9?|2W-}r;*p4h%dzq8KV2P!7-j3;H zU&k(!qgZve>8mnr9X$I+Ctn?45X+D)zL9_a%wOvjfiuZn#T>J0&HFBh=fs)(hR{wi zbQsf~09_59nHQkY2EI(jqMGtPj}Ey;L((P0cQlHB zqOZ$S!np#4Zp@j$T>T*_U+qvuS&fV*2#?ggE2#iqG%ui@!A(SolkQoe@LU=9QtIgT z(;uAqe8Tr8xf^~u&mJ z+Bg&|(p?XdaL*XxnQSx3KkM@W6wivV7ZtV+W92PMyf2QI=ONo>gDZS+_TfB8V^*S+ zEASA{1B~@PGG5$BV%{R1omt(gnCK5uQ&n0{=%71RdwYPti; zKZjKJt&|t6lY3mkW-G=wbnsUJ_p*F&6fv_}Tu$B|ZY&^P9Le+U`*H=;qZ4C=-xA}~ zuEY{PtY*k#SC-)aqmWR^ENzZf#psXRVZD5FrOGVEfKOsLPK-vUnqCi9NB#yRtKQmH zRK3@25txpCXX2yEaUvJi-MKgo$MPyJ!6R=QgrFpqhS$@rc3PeB$fpL7s+Q9vjaw0f zoYX39RXRUAUD@?Rd-ROVqaTHoC9x*3FH=R~`+Ib-V@D}~uKiVkHc72Uh> zNsXm2=aqcJE47k1j2QPuC8sQ{-V)_S!w3L%WY|e!l!SjOD6P+G6hyv{j}UFDL8nu_ zF=701j66By%~{C6T+JPHMA$#;+|=`6a;PMvl(8v>n4Hen+&sEI+H@U2Rn`u}SZBH~ zI}F~(uZN?I0ZouRvcIbr{vO>*k4VdYF0h;=qg1iE9Cv6JVOS^r(qVtK$4b{<>n0LS zFSSTAn-eH=YN^V8v!B1;Y(Mx+NVnbFj>jRdElfa2ptbbD(q52^NQ)d6orl9PL86IY z6x&sH<-Pmb14x0=4dn7un|Z78=JLehqj$A^0}m>C0`y*FuMkgHMXy?tXA9NFc2ck*ha2dg>||dAWsrtt=~Xx<@V5W&gtw_W8ScZbjT$b@gRjR-ra>5C#{!M~Riam*Id1}KE#N^8m7sVP=s+se3e!egB zkq!x?&wKs6!V7$OTa5ynAq8 z3Of=C5`}99XKTinA@aCxhqzp5oph$PS)@N2b`q$Ko{nZ1+_uIpPT-!lLbnNcMba~7sUj)_p@yTg2e72MIWF` z?tkdZiEnbmqgqJsF3(MH{4j*!S1{{%y8hXf+hP9hLX2zCEA!x~9s>Ozq;2)2$9bPT z78@28;h-Z-d`YQH7()kFZKbbndf0^x&6=0JX+k%N&swjquNRA&PKCT!&?Xfx8-C{( z<_mO6-?5u--=S)R*L_Mkdu;1=SDvYm_95-Fq0hmSx1=hSI_VTEV^Z1f7Ev`-U6!xr zH%XlG+r0M{n3lj8e&^hyC^p%7WF?g`kQGOgYqrhu%seR+bD*|>^n59CVn%l!XGTWH z=Xf!+QHWW2Gl%{i%cgBzidpw~ikZ?gx1rq}tJ0zDU_2%MuOfE3U;4s4T_XN9$iRQyJ7sxcQOCrcJK`mh5;E-fK2)a9rlZqb7rd5CYwY~ zR&sh7%S0Yc(g6~gD3q3&<`KQ{xP@e|YTWgr>t2s^fE>5RnZhfG{N>bni92D$dVwz(ggWyOsqVya(I|$MM9S} zh29obxEsKXa@kB80NOf>rX>AwR$T!N-6+FE&MahnJxt22oX09q+>%n@;(qL;u+1y< zI1L9o#==^Cau?ev<$Q(j-RLUa(>((@Mj6M=-BG`eFSNuD>~%7UwE8Kp1wMGEC36hq zs7NDbx00R@4Qq5SlXiqgofJ#usJKi4h>?9~cG_^~6XYL1pHSyD3RbVS<58+SMG0qa zq7AL@B5c6X>mb$ikR7pJDw+DaCNl8&EA1nXs_TKiuC=t80*G&@~5n~0}YE3QWLxSobREsC7mnwaBK0g~qF=+pg z(4L~l;}ux`tARFc#&E`%toklSvP(%eRZPF~r~fW<_@XRT%j@y_Qzpasut?&y#b?;L z6}|sNt8CZOK7EnJ<)M#{P!3{SWb$sD>ROH*9I{ljO}wMx%hh_^C>`UJ?brzbQQF?= zs@b3Pm4==Sta4BKYk7OGQV7*;&sb?PGk!978(k#n9wVeMhw5DreiD^>d^ID&ZisL| z*r&Q!dCR#!b3d$v$PjtId`6dpaa|F2WV2>LD>bW1Wx`VQ&1^;TrcwOr1>~CofhjXx zG6CY(ESa2mc8$1a%rm#hjC+z|=a`CKuza?K(sjap`K`j((t3;As%^bYt*SckwSwP`lyFc_Ag=npaY$!Tr#)T=3 z*DWebmBr1AL3-17Abeg^`MY4Obzj4$#fea3 zYy5UthsvW_qq=uYK=0u&9DZU-yk&j+eFsg<8Zn`bgm~VoNAaqKXEC@2T#wd=a&KaP zuNsY5Ev%DkZS67NU*FA0S$Qv>r1O}F*}Xt`fm!yWBi$EFCM>U$Gvx(s(L0T!DSq_a ze?A-F-8Sb?qmXYZY}|~Wxl|+lF}w>ksDdiPQp%=IUSSU{C)bfCUtfl23m(c`n&<2t z&ik0Wclc%~{|w$c>DKeHQA^EGc)u(iqCQ-lD)J6vIM%u6k(jjgFW3Esea{Sb$uk|L zdBN~ap58%ox*y88=H`i}noaXg(FMU0YASXVvg^fpGe+_3bnXJ;P3HrXD~ASnEVP)5 zx-ucp72`C7a!E&>je0$`l*llCSagpInklXO7ca}lzBEsY>K-x2WjY@=jw@3;LA~s4 z-x8CR$(;Dd0S~9+b(HX+--l?zQ1BF{P12`#MWTlp;RiD6re(c!WGGM9ZIsXFSbsEvEb@=F)fq-SDm z36UGY2bd$z;s=;;YF~}qduaRFr*Ei9o|s2eLulVK?<~&9{ZIPpXZ4^CC;OWxrCY74 zjsr(V?y`J|O)3&E)Z(nVC>)tIO+f-VnmmZzMuG>ouvas_5nT(l`+AHSKFv# zr30%1-Eo{D2Rh|XDb$jf-m!Dr@U)7xX}o>SzhWaguB^#b5{57L14}CU;C$k7ZSU!47J)F$8?_hVX;LP%p3LaFo$ST3+%B#4 zYb1Kt6DsgT!qTL4(_`Ddf@EL9fEhrOI%J|Q|${M;&UrhQT4+NH$kG!trh+gW^G z@XKap1&hkmv$|IORT>>6?rIniads5D1an&LMhcYrxBtN|F`@EODmk??)7~MrlXdiV zEZ%c9vk8ASco5$r!CxTg4|20}(aGW8h@p;aQ}M{w{ELYZ9jg+~-j$E!R7E#qE)3@x z>Aw&L~LHYUv&l8J>XzU#Q2wh|AdI>-v1v7X#@3FYAv)zlr?S~0I@|2&l6Y6DWE znnCWu2$!9-iW{P??9A>~>k^Fl>K`!+!*n_wvbHM?*Y%2+ua{4*2ZkBCS(wM(X8%JU zIxOFm<%06A;)X+FVw%Rh&*|S!FQma)h?S&+)_aw_N*^T64zBg$llt9TW0}KwdWc|I z0>uXZ&_o*X#=2d=2}U&G{_3P4G9dD=3To?SmXJn1WGm(;s*g>njtc*fgqZHlNxU7~ zLoCbB+3dq_KqeG^D$7FoGXj;zd8F>cbH&g1?@ooyx-5vj0}(7K06zoPvGb>CUH58r-`@Bl_sJPE%NbnGoE?O-M+Fe+x{5Mr zVLb-&L2ONl?T-+7Ye}n5jfMEm`s@E*7+L10xj5wm@Ze>3co1T=L6YntCL#+1ItR@mzB3oqzk^3>xJ}G`#<>2fMmHm~)YM(AM85Fu}r5 zcX)X9Cg~F;{#jJ*@8AQybRhLuo!kbo4H$~O;{EA=UMItk{>Ab6N3f;D{QRsu5}{Mq z;73gKsYuJM%PabcQGxgfQ+3v$2OZ(kFU6@VIWB^Pplk_@jLO`}7;hu*N`*2J=Gj4) zh{e9>T5|S180_jM>S1=?D`$wB7~DdX&+wf)#p!!YAg-5{6f=_yzvJDWF;zVm!jgiJ zv#leu0LAEiu`x9+vj|3e|KIVWer=HN1Ipd9aKkSyFWXYk-am)Gk+|om5OQ0Fjf2_w zFV0KKC}kK*0pOBPdF+NUt5USzq98*cg=YO=6mhag4C-B7 zpl^Zn$G`aWN~tTBGhIKvgTnVIl-N03(|wUuOY9F1e`f>bCH$G0>N%9%YjyWT*MWg+ zl7L*^_dzDd$0ck_dJGmpH5Iv$Z4Ag#w4{ZlNVG}(GDvI<7je8vbueN>40*zZy^lWt zB{8MKtSv?S)s5us`L@$RFNw@+ums4Wo@Z;J-oPRMYp`tX%kwmj4Exs?SJ$$aAVr;f z{Z14ed7{-vzjI@ixZpCEp$jl{4>sS_Z%bb}myXg^!~yUVa0rmfU=rBrRT^pL8XztK z*TG;5!*rGwq~E_$eJ;;kKSI^>i^wN%9y8(DwnO~&5mxBr4W%@gb#?4gI+E47#w4MWq3V4wdI=+*{DP#5k( z6cv`=UZXoq6*#N8kU4jAmQjoZ^t_jYw;;iTZQ06~@m@0XE1I?M)Af9B&`I%=CEdf` zUmNW!0r7{l{;jOyXCFvL{D!}7jrs8fGXL}C07IrIefF0%Rb21RUlSjSs)E;NCJn?n z8TwQZIj6kxGnlq}?+qHGcRAmN{#iI-z>ujm(=b7>6YvM)rUAt(Ya^xlOYX5IYltmG z-C;t)|Aps=B?PSzEZ*z?7ax6z*n|A?l<)elZUdAm+_1Inf>Qot3HE<9r}$Y1Pw;7@ zAv!vBH~WV0w%Qo{?Sr-c!ITQeLci@ z{Qo!$SNv1%fI5tO=q`BC{=BDGH3yOvFAd5_t;<6 z`G%J})kYQC$_d{5sY&K7f(%fzU9 zDKH5MjSW1vkdOpBQoHqv7chpi=eCU9V~eh?#+ zxrzAjjRT`2d`Gpd?+w6k9f(uUh23}ts3JeI$S?MQ96Wds{2yAD;D91&adAOFK@8IaPgjhpCVaYJ5c={O;4Ss@wlN@3)`VPd7-&X2zVoVV0MWnkz z4|sDdDZc{}Ve1ssU}g`UaOzyvZNUU&(>(4*gWRiyULZB5DQJq&SVwH&%SRQ8NkNw#MGM1y0L$lviO8|EdWV$Cgl3Oo|e1b5djCnI8sk~FB3^_LP|yo3;<{q(#r`_eZ^k) z%SFVM6=_&;sdtcn!nyiU?m;hb$sS>6268n*9?$~|Z*w6ckuUH>H^wAJ97ZySi2r$} zu60%*hJ_W9IT0vxb$2}?GWYiA5Z@n$v+@2%&|?vFzlw!<{)9?M!6n*#5KPU7rXk(j zMg1rBUnLZ8-;Bp-Afwg7NIQOJ6w&PW0P4T7XApY7&N@ld<2h(gUweOCIi_tRZm~fr zd7*`1mAb8pC+c!&cnsVqS{?sun>G~Yr>S>+MCIO?lKvCl?f}G1bPRkg0uWF055=zx zL?AgJL@H}v+oNf&6<4`9!qOURX72m-a3t|Bxikvr!db%Ihu!2way3>{{-Jw zV4o>&6oy!r2j@QQ^+miIzyV)@J6ho8Gy+m0T|mA9D}+Sn(TgWHQ@+B9drxQ46Hz+e z7~vooyK1ooLlG2628I6n=SZO@eb8TXoqDDvUx5c$#fzRmU@QYsf4I zxIc$|+#b;kdQU@*TKq zzcctO+RNI|X+qm-m@`E= z3zS1Ixb`|(%2`wLG>785WzNA?GYG%u72-kq02WG0bC+W4n{i?F1r-0sXtMWgyp~^T z68k;{+Hl!ru1{P*S`_hEaPC)dD?<_b+VRJJ=yPRk`qWDyca{&*;-hVD5YT?rN?iU$ zW#x+q8$4b~IZ^JvhP&Q9i1)r@ zGEx1A$qY`s@sE)Bvwl>DhR>@rD_{7&1Ghcrcs$l)i{1kOu_A?hZ%o(+XN0>69f*^JvVEcW~_|LjhCmLEtk6M`$T3Mxo0-L zmOr$R5}~^1Qi~+nN6X+#Xpx}yh&a9$S1Ub;vyC@3caOcg?%(SQ937#j{t7{_NJ$ka z*^_1cP&>-+lqR&ePljjltF{pC#^Dx=E$!l&za}bj)X+m;@wn zFn3Jm!#6R4eK>$=T-@)$Grm<83nYtYMlIMSFW$=YFOGX<3uaLH`xr7S+R_f=#N>); zNn2bNj^CN(Ctm<>R!CYst^sN=_Mx3n+#7T@CS-S(vo)4Cq}bC$9`M`k-$yF45jdE; z`tydelwp4Q_r3<>vJlVH^1q}ujnK~sFfuc9<2fYab4Zmb_oqpGnj+*{#(l5Fd-em) ze6C`)sD)&(dnFwVIPrmYJ?O)$+&%q(%=(q74- z`8Jy5S`BWKBiE|CJ0U4@%O@YQf(I)Oi=xPKnxidoaVjScD5*rp4)>(=D*3}5f!38I zF9=Lb3+N!`To_;>=ZDIQE|$^!*lzchS!LJjD<4a~b=+w9HuTEOpe1MWD$3&1Zx4m| zv?(|$EH`kLF{`ydpE~`vs(x6berxe7O(KuanVX?;oKbR@h)x2k?C)nNq!bRMT~1y{ zhtC2sWHSj-rz+9z*w3wUzVG1>O)21$E)OJ|TyRdxGLtp7P-7YW+;lJHF9V{u_yCO#vLe8;7DZK{8c0vabKyDb)RF zsnvT&j)eRsHZd)*lyGmo)rMifrTT4T@pj7y?Kt)6H)gxf*CeyBBYzG~dE(Drpg9xA z$D}j17|loJIa2FORY#A)Pn@AY_p0e%K}(JeosFKCyqqEEi}7Q-DrNLMGUwYt(5FJ@ zu@5HcCm1$vQm41izx|@%*L*~`s;>IlO@si2M|eph5*dD28xrVG0<(pttiQOdOYBY> zeE*G=elDHG{Cf7zEN0L7^Y5rcT9ndspJ6hbh;&S!@eOgBe0yCmxg!W8MCT7OX}9V1 zLiK#T1gzdA^5%QsBINK(u%RheUwRxXCH8?ZaQ}3mg;Dy~CvJhLvXpmF#=f+jHyQ)K zSzr{ig|7Zg^%=aY^%*d%R{spb^%-wGKbN7CUOByF(oZBd35hx~e)BC0)N{|hlA30i z?CxKp?ofpycX@Oft(<)PiLE4?BPsK=0x56iklW?U5-^JfD;2E&W>Jjs>fe0pxcjgB z0-O@1oIE?=3DGuP2XUOM(A^)mnBgkFdl@C>gJvF-gS2D8{n8JW#as(%Dy&Fr9xYb;@5;;u415IsMjI1(itBi#h0-t85DnZct%~e+Mu#OgaBF$< z=?=lK*jJ2g_>s}=N&cu9(!Zf0-cJ1}+Q&qV)ydE0Ws|K}lDfzv-#3)%RqyOw_YQr` zBP@Mq?j^1)g+DNvhjzJZ(Iw{{Do{=P=5LW+`XaxUn)k!mf8c*EC% z5ruA`u%hXYd1#-t8m$U$eQ*EbU`Iy3g^5#pM9J+n0vszEKKrpKYmy2AS57TqxThwQ z6MX{Gq0~Vrp2h!s61GBkjdRxVBn;VR<$Dlq*i~`i3fu3Y2wMq(vt%;9xxM%i zf}iD`BZ?pLhQOH?PJ@Umx~cz8gG^R`z~DUYrZa06wFCB3jJyh^Ick_6IIKLfW|MoZ z*Q4>I^f0YNvskMn0`cWbcj-wL;MQ_C?|qT(=N0le zl`#|%plA2O$QsvTg>h1*|a z=%l{Gc?~@`?C_#Hh^sq4z}Aa?$fD#ad$WS;D6EPm_Xzn$S{Y z&y8pGVkkcwZQJ8Mvi<_eyZ#dvj&S+6ny--s-i(zyGAfBHo^@OuDik!&PQLpEECb)`~+N0^|6hJFDq)(d;3upyj=i!`;)=cJPH=`E@KhafR|KA-dZh=cc` z+=JWzT-jJwr-uP>Dk61*JtisAkJuFQ-ks-nj|6PsjG&2Z6ZIRIRU^a=?D#jh2(ZQOWh-Dv&y@Q}7Msjj~W$)5o=0p38rWggn5EMA_^Tnd~| z0Nt1N#O58ps-=9$u`DGSgtVz6Qhrf|w$13$f*MpV3)RswL-ngP`Ykl6gJ2i9VBeq$ z;z%8(>Y;qK?W3-zG!Ml4hJFVI|JylmQqyFHy^sj>c)qGW+x$A^uWKU#C>>|K8Q^7> zrz#(X-jh%q5t#W!0eKWn0=w~j(X%Z{POwMKI*7c>TveoUT7|wSPCwN6eB*C)hGqd~ zg~z@vLxvj0^)d5U2zx9=TBXeN3Q3dzHpMIAvr3}SW}-3PL~#VAfSXQ%H*O&DYmZ+1 z)lj`zntSX@DHT^Du_B!y|H$p4>}BfCI{xF%F<)X(tG31hllckhZj|mk0eAb?6Hbzmn?oTo`RN5X|Nr}C z44j+-G2#~uUZL!RmAAkQ<}uaa*4tq3YGt)2c%PsHW|kIZF!V%~zcf)ZE@j5(rf~7U z*2N)MFOBkp3OZ?8_e0L#*JE;IH&s)yY>?GhgB@981xG}6%d44ls-~gz+H9M__j;ma z(1juU>EtILU79`!fXI19$%~5EO;tercAMa~>?X|8B7vEkmw!L2`Y7Z!9HO`EF;c&V zMGV)wJNM_w6I1}$adQc$t~t1pl1o?_~M+*94zv7?^fbynzVm zXwAv~pTimdssgw>G`#pbrdrT|TiQvV#x(fP12PmbJpD6yG}7ju$9^xJ);}B94fNlg zCB*5xb_x7IQ4Ad@Q)r>l(%S@h%>5ZAcBH#ZH%XM3De4C`4rlynC{3~iVG+fQiBy&{ z4sIcw#t=mIFu7XN<(S=#BsyZuXpq2eb3#rRl0ieF>W^;TnSF~)5LG&riS|AG=h+U0 zrM^4UsH^JkG6q@C;QR{oIr}S&Jo>^LF=*&3+({xlFobS+JW<8)bw7hsC}1 ziRxXREuiDvfosNYD$5W7F*)X=ltU{20Xn}AdT-kSI=PXL2jFHS&{2d^Hy==B;!)Kz z)vqA+z=k?vmO{n1gAo7KXRgie+?ghlk@DvlJ>L*-$)Uhcheg|t%!@QNkFezh*HIcO zhCE~vUvJkwpZ%s2-h594%{3(IeLM(@qe;Rkm+pPSGW4S`)-yTHOZqM75*TK5>f1^8@{SGV55qZNfO- z?Ok5mp{*$r5?`x#zz!rZOq`hKoMc9>d17}|OfvH(z6|d=fN%S+;E$@O$sMqjAB_~# z?~6QDWg(~OqK%$~UAjA^y}q5KUJcM!0<)K?(#sDV=uu)E{vc?Y+3h{}JI>AT+>CEQ zyEimDG8`bvGH)jKltj7sd3n0va7f$FGdV*qu-Xzn)btcy(%4_`#0QC3^*hZ{x&4mmHK$$?a7CjN@`1 zoPcm5J)&CA?1rH8<}z!E)8X zgL9Lnh*n$J$;WnfSn#w8x*6$B@dP50CLlXFPz^lSqA z(JI*C$^Nx$&3xIO38~=V#tf_*gg6J4qdhzvk9p!IWHMNtoRRsQeDtH9JnY0KY8+5cXfk z2Ob9#Wp}xcz;-x?VHu3a=v<~8W%_DsjGu(Pb}+|H{c)RE zp%jZRUn24cMz!r8^9#C9MI@3exjT9hZo0Ni(MIuHeW<$!^p{Wo z6nXRY3x|zNp&Yu~eW~#7L66x2))(;#EEtn-=@{jU3eC@mX#4(?bVP}jp$^f0V$k>w z_23x0`1GAlJ1wnpqhHRwoAR8Jr3mOtegirwMjPXm_7)DK)s-Q54D&;7_<2-3olo}3 z8q(r@r^f(2q7t;vh3w?UBdTHZ4%0wA$kU4QFX5arHP>(9LV)d^cA7}6u5Wg*`~+^R zoL=vMJvh2MJqa)Wz{Osv>=VQjSm>y0O!b7TUW~^MB00D z8Q8shri4NpMS3*?CvV}1qjm<*j@sWI9pNJNQ)FJ-Uma$dED~PSh(l9uNNTB2gfV>V z-Fbk{Y5V}SW{%Tpkd5UZZic%>lX=j7XKO`j1)cVFnToVn!|^BMi<;%K`kluWZri5Y zKm>Swk8LTW4SM#8O#$~)=@hUbx}hh7$K=Z`V-;pUKmwjE>4!8IbPVBwL&Fc}YQ3^; z_IPc8W)xa&ij>-E->T-!?o!X7$;)r6tRo>Zuk}l`%hOd_v3}J@kO^T}gkjm)u2e$= zcnel%p>X5m;h|uMY}Q=)%52ydDD7)J2aquoR>|;jde2R_3+*C}4yGck$T5YrOdroIO%km4W?)~m$?zX~fS%0~6H2Dd*Meo}Eph*$;W}Y|MvU5VM ztk9~&`gby3K_9zavszRvOm4hIuS|Uomq@k1>#*5%c@dJ4(Y(Fi_D0;?Egdx*9gB$Td(LG+LsbsuqXp~9 zPo??2YC79*?^|?Qtg7F^Nl7e4M_WtnhJE($mc)q;7XzbD_TvMpjT{~2!iHueY7b7K zsKcH$MDP_d}#z2VFHewOp&B?6;9my|4{R4AS>W~05y4*QJu(G`=d zS;i~LflUbDZ@Z%mi}HDo|k z&)xo$%~QIf*nYIt%~v9m$r6IQrE>~p3;7$s4Wcm#?pN6wZ~`v%JK#EIE=G9KDbBpf zl=s3DgC9m$b!H2{jp*$sBZ&B4^nwpP5nhN6$>LKhpK}{ncXmS0zj3c{;d_v4wtls? z#v%lkh_tyUH+R9$$T}}v&mN1Bo=up53)J6ZNS8=~&HKW`9~GUKdta_H@wU*MVC7^* zn}n0_B|JX~iOpmff#6+no;El2&Rs4u8$mJVTS87It(pcOJ%J{X>9s{lX@YWuv>)Y? zK069Y#>dRwInNV~?@_l(W`v8_Kg0)#3jV3+Pt-7}O1i)Q1cC|koe^dSK&mqq;POXp z!uUb4D}l{E;#rzJIgS4oL zg$wDX#^F=xNb*u`>L0q}4RACH9ZHk@cGPQsJ*=dY73^vDqYoC4mA5Cqq!@k6Sf>;U zt%?o`R7aaeaKip}x>U~~#PeuzM{VH)=!~gOp7kCx4WN_1Rac$(mEc~LID$=ui3~Od z0D(|`z&x~@Pm`rgYOZ(RHHUPMOk@tU&IwJI7oN}k{aEnm6`rqh)aj;jE`soRm1zc4 z=6*tpn>_o4XBDLdyn|ahsu{wkipUnYPVd*T2N!@Y)%Te{zqWKgXaErtZu(yta`#-R zx8#$rhvlWWTkaTH+u}J+&IK>LGRcvAb4y7GK%BaEk6F$)B2*P9cHiILi9SyD2Y8bY z1q6>dQ|7UXd!8Y~>^P!FnJ)wFMoOZg!QLnp27<4wIRPDJ_b)d(5%1Fpp0yo0i|?yU zCGD-}uHb~lG9GqLiX*r}Ii4wgE%==4-&-qR@syJ^A`-bN-Hc@XbyT2#kMudbV_656 z#JcpFAeq9R!ZhRmjCjF ze;bGXve+~nj(tRqPxX%1ZfS)Z(xVTEa=e7)TPogO6yUq<_t^CK_rG%rfAjAyRjz(E zGktO)XOSx!e^+kA4DIKZwd~e^qDho1icWrr3xK#T~jO(CFQY>N-&AC_~iL0XnJgpdLh$B8SZrZ z=Lp=zMc2O!%T-&*u{oKx7o&qK{$F_RMg5KX9NLl!RHEV;L;msZ4E$P9)`A=?v^*rI zDaA*`WuV&it>X$8RzD$d5a^NCV1kFdTv(p8X#nIoL3b%Z`%$Q)-W`jeWzTG4NP7|E zfJ4dzJ+=axy?&T$v=F*o;Y{#b~ ztDSxDHvYW{E-~jO-Pnc{J$;yBh$l})8Y)nZQ-jKb$Vc=L+|X-dQ3`o%Rljp~LdYEo z^C^WLl5|S1GJh?WPsgnh*F#Voc}}2{2*&~)SQD>5JKrB`uhR*`@OLSsje<;t2qcf8 za)U*H`*yWTF^eCU9|B;5Va!~wsT3{>GH+$Z{H=jk{;~=%;IthK@S(STF8h*o@z+p1 z*Et@;6>fQJLvMYV)H*3%J5IOfmfV+^V|k-1OnI9QTU<%H1|0eMI;HfXQ)>#n<~31j zz3jqIt~kWKwqe^Tk@I29=BqgjW(xobP`8*gZx3PTe{(#_OtdS4(|s_BA1aCUk=1f8tIHyrUew{}t_ep)Bq zp<3DltQ~mdr%Hn|j76*>Y#7Z>TkP2aByIw9H`nXE^dF7EEzuM`BNHACh*( zHwT^{o8=h^;c6IJwbBKu@F`dlml#n+I)AIjshQ&Mlqf9y19v+Qs8KkIG}4)7PqqOJ z#Hk@2jo0Q=0*XQCZj9{%xFCcsc5!N?wBV?RR)2BB3zJzfau6;MQF>L3_XBWUPqwZe zntBk)-jZ;X9emnGvBzCQs$XpQ?RIR7_bVp#q<79$GAugR|E{n_-51za{0r`qcmY9F|JT3& zWCVc4H~W&Q{J-^aR&}6x68t?J|KWN5=U@9T{!oAbPd|OGIO6=5KjVLXbrSbQYC84( z;flHc`XBut{}8YG`jxkRm7B|Mge#EQZ79 zGESvf+H^16NE*5aaO@u_?PC60J5?6#5u$Ow;~S>@;SE2<|M{8yzu#|JALh7~sLmbC z@KeTr;~)6<&XjG-zskM@b;SSm=YbDkBkw*o?Y)2VaOG#A`XmA1qGn~RByj~OC(FhZy(pN}7KuQ|FkqLX!ndk~ z!_2?Wu@?bTK-s`-+n2^jD7FyS*PHhpV#Ro8{5GYwK}H(^1d_2MX!Q^>;pnZ;?vmw) zU$uBVgU?`F+29~dwQ1b%0=O!^njQ80c+^xQZ=7bWH(8N0sRQgw-cV?aR@Z z*ctB>b6<)G#iNJ<36Tl+-FgGr)UVJGTF3r!BafHO*eijqMjr|9yGiBpX?}KcSP2uh^-goxVQBI6#9}b+V^C-5X=<(cChL(Vy=GY69q;VxC{2#7 zyj|l^KbXj_JYD*B}}H(c8tlxdOrlS@y?Ru*m1i#^FUiENq+FnW!N9V!?kVPNvjU5PrF3#cOelZCMqH;XRt&)A-@+ z4XuIo@%}H*Q-?~eS8{e`zUk@Eq{qtMUQEqsPKBLXmy#35i;P0K8~)ghTy0Y|QF=8v z=%q^TG|3;(k=sCwRiT?!0@6qIaO%UHpsF1hyO;o7Zt1i`>Fa9RAjxj8rvt`-ao2%H zQ+;uTHJQ;s8u;iP5P-+Sn2&UWFv-Y=meWaZrM1!p6aiAw`&^>0yub0yl#pKf4rGGJ zBZH|1-V+~Dcq6jmhtK;@SajyF3siG5r2Xqt#L~Wqft+CR&-VNxn2Z>)i0H~X?@+dZ z;Ml4nNB84Vapt4nqNKZ~XlvAc#ncg?@5=C+dRlv*u7ubYQ;_o}$ZP_Vq!bZahKYDjk$LWE``jN>e} z=MCUWQyr{Q`|s1YQM*I5j5yg@Z5VVRC%Rl7J$NR_d>GQGd{w|3K0Jm|B`la}OH^{* zRjF8!JP)Cr;IK@IQWiezP5Zc zbHCvWOoLHb_p_?}2}v8nZu>8uT~Fvu?X>VkTt)*6edmidv-lJ-P|#RJLu#|_CBDi*0!n(5*<3%&|*FkWr`UtJQw&a z&l{`}d>t7dnOaBZ!DQwi@-a&Fv8atX_w#gvF~U07QepysV2I`KaTmefZ3!F$12p@Q zm$OfFOY~TI=u^3G=`F>}-gs)oZ9NZ^9OpTV3>EM>(OM`)!7GsfijZVi@w-58q??u& zi%xkvc*u)BG{^5Y2*4TIG6~3Wy5@(eP2l~CAF#qC!k~C&Wwn~k+yJjU2H9;fm{w$6 zW!s;5Q9~rGGhOh056q@i1gxyB1Pfi&yPjIHKqsv8@$ca!&Njc#dlbs1^SQVO#Znyx z6H(1y%|{0N$(GTTuJN$IOw?oouDVBnVP+XB^LZ!#NVNPZA8kvwCWa`7WS<|v;a1huH{;uTew^>U}7&RffR*g-{?o&a%@X1>}Pd&9UVWVTUS(kdd&r)E|$KFM`AvIk{9bn;I_B=DOHu=ybWL)2Gnr(Ea-iD@`aMvVF>?TCBNh=lg*u+^OKi^R)0V=po-rWN9@)zX37Do z-lJlP%R593o+PbUj8`>t^6}A%2`$`+nTA~yb(en4+AoQ0STYdUpa7n2R-cWP*cjSm zoKdE+dNF0DyNsxR8p&C=2!3Qf(gFc*ydL=JtI!;PB%Ua8mxiUAgeq^+d#Yz&vvrj(6nu+EOZ zpNk8iZ3f%{H|6hap`ZCSbllo;vX>Yx+~>+CQ}Cpy$CsaBJ_iN;i^|UWH~R&j=efpW zR3vqA=FJvDgEzn-dpSdj-allNVo$#8k-{lGOzpO#L1Y*xTBBpxu%%^1(&7k=kr&!6 z#8{JPRKbS*@U`*_pO}JRR3^KS-^U|F1OI@Xv}U+a27O3b8;0iEaq|y#%)$ZJW&?j_zvnHkL)(Wlk>4@s)3x8P%}=NBbjNW zG-tw(bAn+~rI|;0(=D^T0Qqd)xJXGI^Ufb;Yn^>0-oX)%q2iv@&u*Ep1rIfYI5< zP?HN-wDjgqnMq{x7dke>QcsYr$7#kHT|pPIqTcH$`z$B6?w+I2>YzMI_!Y->MFOg* z&!Cf!a%qBV)K+WyrgobMjoeSO^P|*Ju{XnK*9&p%CBwoM97%Q{yDcEyWGQ`PPN^!g& zo-iGDHE>2gGRTC=((5hu%?kRim(Wjw>eWxG7 ziRkH4vc9O~wvwnklfHDza-oMYoYwM`cACV7Ve~{#`gmog2;EN#!2Y)ZnnTNcGa~-& z!;kQ(Oc;3!QUI(d@fTVwwb;U3gX-!Ew~%DY+xxwcaPYsjljP5WL~DQ?GwHQy+vXt= zT~dwua?N6Is)Xkyf6y99NiJUDHX%+*URKSOCjpX^UUE|@F1(mN3FHdmHcrMQN8?t0 z*Fko+AL&7~vE>8S-S~G%7P-`-z=6Imj2V^uQt>Wav-PZYV_y;qEs;-Y;lMBi%a;7_ z%Ov?IfHlZXZWb=rhaN#7A{?mK-bfbUGno?AQ#6e2x}(G!SrtXJpuG@=# zZkh-keX0SNXn_N-9N8}aQ6pLa#eCq9VGF|5WY9bsgFLL2hSG;GR^J;7Z-Ri!Cj0)! z<3E78sjfJJbffSD?#Q0$@BV0h;`NIpwe*(Pa*)$0XHeL)(QT7gsm*GYj)+D9?G{N_ zs_u17GO5EAW=tPsM(&*|+n6?~*tv%wGofQ8m?LwNr$Mas*=`S`xwLK`24VCtUObqQ zpuEt+Vf&nTX97P1x5CSzHcZ(&Gs9nI=w8Tt=c(YT|KQ0*#I<2?zOM^0T=L6Xagi{ldJgOv-TsW7;oG*m-)Q-V9iU-z&uDhl7D- zy+<|*ls+qpO1ubUe^PmvEdk|kmRo?I#mhpJU4z8q02l$Y_lW*u)(=(E;wKm*1={JS zwv35jHK$#B$qyz3WjZIwDs($N`kh^Azd8@x+G;hTkxj<->i_3tnq9^%JugsAT!84r zYWC*B)O*+XOA@J;E+pRDx^~_vbH<1`D6*DedR@vE!dXVqix+Ut#Qe)o94_+=f ztoJ*jsRAaM2SIvl{GAN#EpYYXGdln_Gh4`p7$-Z)%V6SD&SGE2)(Tnhg-*-=d^TuK zD}pSh`=O&6-X*eA<@e<<5M2$_$el;GhuWzIkPO}(5FHIAMikNR6UsIQ;8=`qIT9D4 zCGMGYU${g-lV-I=_2oHX#}kBUsS2UEn)2K#un>l_sUW5N&g*wGI}b1i<7HKeyTWw| zuWFAfXpi`Z;Fo~bv@hC=zHW(w}Rmi&exg;?4V_KU6Z1V|Gr1m*~eF{Eqz$9EZ*?b ziL_Pb!`df*L?-lu?4Ku1Vi1*gP@qa5bKYx?gD6`OmGAJSscL(xZ{{oZ&R5Y&AG@i7 zSYdaJqB9QXrQqn5zNk%8s_vsOvg2%C&vf z)k{sCzJlVlpT^gBE@obWhjp1GfUhNuCEE=+c#{9#9xFEe_||bU|8n`0=DQ-JK0E~eA%c=ie>5ccuhc}QSRHji^M!$C zKm3xou%n?=6iEa8{73LAzCaeo$cXY@_a4c>UMA7IfOgjO_^?hi8IiIJ?a~rZwpDw$ zC=j$UpY6&(x&sSa{V9H^b3SW|4zopx5L=lUWLF4CwfGZ%zRW8m*`1$+SGFJgauPUe zcK*qQ)D#}mr4T#2sp^t@p)09G%jHZYrG(qdGon|x==u-)=HdG^4nC+*>lPnIFg-JOuf(l~Xiu&y{R#YodaidUf zfx`5S&|S~iLlMdPWR0%EJ5s?}VY}!!N$6-1*DX9k-_Ol{BNGGSvsWTU9v(CMj}s#g z=ZSJT30taw0H2dw>oC<|=PsxgiF1%kM;;<$hY$#;;U2CpE2XxJWWDp4#bJ50LX*v68+r`V@OLmGL-f@E*+af5QKp#+im=Njs&`-~m20qI3@#$opHlWC%58y{8r0AVRWBUlQ(wJsWRQvB-=Fm9 zo<&^&C@uy|@H|_CzKg_A^0(@X=o?@2ps%=%>+X{FpaHA_1(AJ0$*FR?7PbC1(xe=N zNDBg5LgufoA<-3@ad$T=5i!O~yS(EF>>Cr;365`oXuZuBPoS^gDwd9fTx@?=GTZGK z1cyx0x8L!0hAp5RV@_|+OW}kqJdVz9+#why(%Bsg0DC_A^R@i~Ec6w5 z-EN*`f~G#cH@co%h^Z~9kf{l5a~f)!4|_|LVyL&5l|YckcX0(@3G0b8IlG*hpbaCI zyK0)gpyymf*R9&0;C|mu%908E_|lH?7(3jn>ze4ZI^&QZ(*Tjui+x$N7j1$*k{PMm z1=@K~3|5Y0)~O#HiyCu>{3U-?kJ9>t96qcj&|)^(yIdXDS^KunSR5&*epDR+OD-kf z!X|PGfag+D3DCd`3$cE}^!U##aKd<*3;4uh$dl!|v%audn?Wjkh@r4kdkCfcmoQdJ zw|FAPyQc{fk9qb#RjYcgjtx6Ksr&%uJZ+M9Bh=*1_)cD%ntaeoYTrG97^{EQHV0YT z7pRG^Uq@iDhaPm*tfStPM(_O2ux#%4rOkEU>yx%*MBM*e{b=aX z$|PYKrO}e82s0H~GNi_*vg}Y2wWTI%qcbiVCavtxikehbxbPfwnuyjUb^&`i0&%~% zX?_0|5O7@+!2B!QbvD#=d6yMfDXXH1TR+C>+5-=-bymD(jF^tpvl%ZyNkx2HSn|83 z1jr_Ph8o_dK}YRb7rF?l&T#!8oag>lpYqM*PyFA_4Z7rLG`Xcc5%f1JHxEj*fnspv zmyZ{urHX%fDlNG;bko8@=GhPy2h(V#7syEoqKl6=~7!8q6-aC(7CrH-DT`@&fE+ z8gyM*l@o;QsUcEh8WOJ2R^GH?ZUrwLZH3B(_DBeAK)mENH3`;gt^MVlV2epNRR*4; z%lm1bz4@YH%A$ewy_Q6`|CUIyuk;Gl>R+4agxWpPSWJaZO7U@RdRQl`dWu4A$JLQF z$UTVZ+nhm>Bm!`9OvrR%XHXBmhpR!D`L_(zMz%yPS}~Sq(#n*dpOOwRKmLU%xV&zK z(|F)5mX{ZA;DW8-$MpBcTP{U$D6$uAcV%P%&?N2}=b#uH6w=&Vm{Kk)fbal=p76Qh z^{;qhUf`Qi5Dxl+pueu#2F9;%3tz!HxbOx}hl(ECL@}`Dt_avB;K$x`%`Vr3hc

v59|6R(EIevR&DT~+QcbRdLXwYnXX;#p5 zhhGRtIC+pK^?UYGziT63^@FD`;6m|7YSgaIZkS1Xrh1Pg#$otHXuj94@-{*@aOh>% z?%?4VSR-Z2wC))L%8>~_V>!jXy6+BtVv6*OMgXQOm{EL$nIcXZKQU2@(SMgT*BMOP zd>B+i)0!SMbe*wR*FG*BPda&r+}9Vu2~)wo=!l7jy3QTbah5>*^p6D0VPG!R9NCH( zLU@`-8!})o{eEn-v zliAVgs287_6abczDsAb$37w|a+dqV>rD8eNb0m!7(G-(Ic{lr=yn!CuV!VKLEq}EA zZp}oAKKsjcpktQw{VHGXfEhWW6-;}&2UxnoV0NPUux;{{tEfhFc;>k+juoaWr9Vn6smOkao$`-1ZspwCs%Zm<=x!F>Gu!g z6{jH?>9wiy(haIyz81d+aEX8R+cHP-wTE?L>_aEdtSz>9SveQ{$q^z3G z=6rTT45Nht%8ninR%N>k%VT!GK#+>q{Q=sS_YMN$i@&f5rR%5BtK?$fELbMttvP3~ z`Bwg7PAYe?0Xo5Ee}YY_L?7`%J73~NLb7`|YsbdB z{UZ&JY3&#-m@u;v3DLFhmPr>wDd{+r8#Kd9T1sn?Cfw0QKqCSbIUScWf-C*gNofO4 z`*)$Y1F5*>(0;NHy)xudrudJWub#$eU?E|)+A|ZxBVk{xZ$U<1ggo)~npb>`Y+YmGqVw zIVG~~g(T&v>L5rhGg$z8@CbbQDYs&N64uW2SlaU7$agRewkr9Ydk_Y-4=T`)NlDe+HAtP$wr2ggojVRXKW zPS0n#ZT7He!wG0?O~-D~N4snsnMYJ>x_aEOncGV*xmj3WBzD0BE!vCjnVUiNbLuZ zS|lk_I~jhLSXcY@9z8320!Hk1Z0iR}&sJ^!W9KQBd56$e9=dPpKTTfR5TLm0p?g`t z-yf>;O1R@mxyGQOv^NYxwot(g-qZ@ZsJu*-CN_yf4}ucZeySWug(ITg7^tjjIw6O# z;V5B!WlWNJM9P7}0DsL|7jMWUe8b`8 z6hG|UZ1tz3^&g}|?$u}_X?^;&K)KsQEmon_Q@Dq@LcrA_*O*b!*YB^j4mhNi-|+pA z&&W%nk}6$;Xp=WNTl1%iK;ij?GyH(YG`mh;+EYn1`N!ErEVsKw+y{bfg#;rT%&!H2S-&MXF@-oZRt~Cyz_r-MCTBSM;HYl{ZYf;#ScE?_Q{GHy` zSVo$4z4Atitm`&k!S-{j)QG3fyya=}I_ZIeod7~yb3(M3{-DHr*ExAE?x7pge)ven zjnSf8cM`0x?sekA7TXcRxqW&V33(@DGbuEFbVY-2hH38_ve6$Sn>0a0@(M74iPF^P zsV$@|aRGm2*!g&45c*Nyl0Jt4FAnnU{%|y$BdiN-#bHk>kv8*h|2Vl} zsB0qoBso{N1kCt^?tA1~rAU48lbR?S!qf)*rt+1E);i}2Pw#+1PhfGUP|$LiU}?60 zg0Rva2zQ%}93M2iFbyjAc1xC`9mT&pcTk9mj^uW4!^+m12wcIiQ+mw+uVO<=qPe-a z^jUk09js8{mna8cQqgAdFQ&mK zDtI1FY})v@(yb5s`1q7*PKDjIg%uhRDe!~}+*M=1VlOs)M-&#n=fyxl zkm&}5bc0LXop{k*ti*-;iS0CB%zL>!JU{vy{F;wrIoWF`%E;~9xeNj zhOVM#&?T@9q@f+%sK(9{Qlq;sv`jPDkEs?%05*CnT~GhuP1s*gk%UFa zFZ;ER-+@G8y{z>Qt+TiJ{rlnhF_;jKkLA1A-?a?`4@sUNSf{Q^MVA96zDVLhGIp(M z2U^;Z*+%N>R~P0ENyN}Cwq(FWIwW3^;|-!;VHeT-Maw7HjU>DYvX>ZjaCf*=+pO~b z-df}fFZ-O0WYH!pTlFT~mU{g>1-UxmxgJT^KOyfUESK)|G!!7{s!JN`pg?9vB+0l4Kcv&qmo@_3Km zom|{ned&>1y3;2wQR(-KB($>sYvnDyg)n7yTVGX&|q`GqPpVvz5 zY8N$%KdGjbJxvI&ofq}mxWkiM5i-rz2qJYiC^m-{NwQlXAi$_D9Q36lQ=sf z&1R2Y;}2Hh<2@23C3ly6KgE8Vx@CG@%w6|qS9ow8M4;~sWsYO>Uxi`29y1SVUZV9$ zwUu1nd=x`1yZ{MB{yApSM-g@JAkjkfm*R>;*Yo0qd+&oEQCNOWC>GiF-GP2uBfG>i zW<#^=)ClLu&;!x(^X|d{x}5sSeV~!Z5*!m!U-~GpajY8Jdg+9H_Q#FQe#QlKJ!?YB zMg!Kl!q4EzSpIMv>=%4T#zn+Cy~b|1^=|uW)mOeJL!~e|FT_wR5r^xlrWs#zwJIVF zQ;-al86+$75c)lNZ4j!lBt)9k|2jP!L+|D+2$miNtT~ItJwUH;>u9*?irvLVKr1>_ zg6?S%>LvoLzQ?;uO4y@jv~v;WiO9VsWsVG~WHqu2J5<(Q**rO1O~g-b-q>U63Wpk1QT`CfuFN{FCMoi7r$G;*ZBdEL8K^%ewPRmy~IRLOtof26~_GM?t+ ztKb`<7GY_%SH^*UPx@%0&iy`5BC%m-$HRHxAKa#%J$h$9+TqKna~bcRHpSm_PZAV5 zPJ_>E=Sj*TdQjYk_#Wdmi$WBMG|0kC%^$S~iX57s?C#nrTV%R`LvDtst3BMSGcy1t z^swLu1>hvkEYbbROdkq!j7Cr2F317 z3U84_H6~^Uo*fPyHKQ@|>Hxo?7J5PQ+QEbs(Q;PUi#P+faJR5!0}-wu8exZpY67=q z2&|9%*6?_L_#0)2v~VR!Cz25AKY1W%L@)R8eeb8k$KXXd{@k0$q7#q9P)4w?_UVHV zj8xFMY%750dvlC#_$^mT;e`uSO{xlV28ezI zDP(*cjLp5_U3G_Ki^pqWbT$olNPaO2QipJc#O)zQl#eodbYUjK){v+%jZ<(pz^11K z_OGzf%V&89EP1GoRTjZS*-NY!vbq#ve8K!H&q=vFk|F${2S{gWp|{6Z!(xN|F#6r| zm{vG#?_p~lTMXz9m6X52{@uzcbq#D-O4nP>sgq}Q!o|4r*DMHm%K~+OZ;Rt386#?; zK3ee?zx*N4D5z`u`tw5{h1wlchWYE6utnHfBQf@$6pw$g!?P}nSB@*36fLMbZXf)K z#G3`>5Xj3R!4#x>bUa6ebStC z6^7Rs3|!`M;p_Dva${xtFqdn-5GuyB-mg=$dCj^H^q8f+5Q!mBZmJWZ!)xRMlf%?H zgEmJBpRscbt~x_j1%i16C^LT5VY|j1?Z+y#8gY9r^{w=gC*96f|Bfi8z234je%iT4 zUvSi{ojtX(wTl4_W)Hkk_4tAMGfUW3>`2jg3yqH{gHIUYCMWS%`x02yJ>GjZJP^?` zAEKI`TO#=()w6<`g7E|&>Lv`3muj4V}W`a`wnnCx0TFkK{i8wDDMLy6&`d7flr2c zuGO`%-aNxlbawp`{ON)%eLIAR*71tv$9pm z2)J3z)AtKen*Bi##ZVuIF3N`aw7}hG<>*&C6uz}lt^xYEyJ^rd3HOx6@4QWmSP6H+ z-f6l=L6Bgpae#B$N*i$JzK}izs!$GUTQnK=kkZzWE8ItX)p8|4jCh@p?66YUJ+U@e zcG89+%$BmX^ZAvbdN{x0d#?4Lm)?5fPw`!hNmgdFSR|1gtia*9XSwi6ijNWFCJTN_ z+FnLVg$yJ9BkU3z`4=zN!PU;hTem=ZH3jYWd*g1|7WoNjY`l}adz30|r1Flx*C$7B zm7o`5WfR&Nn|Mc+i$ClSiD?c{uVbMU6!%!kB?!~;o#R2s8tE4blL z|8c0Hh*KE*Wor^^@&!gPTNxKQdo{-S#0672w(3x&5R+1G3Rts8$5~+Nj6s!nR70|Z zE6LbIl0oe+A{p}-3vZ9y4Q-}=V}X#!X~AWhS>9jp?}KP&@dD1a=K2%95kA^hDjYvc zcvCl7t?}gBzp||;5A8BA2jgdK)Xz$x{A6XGy2}?$hdQ>?FJLWPvgT;8i2u$DQB$MC zUw-u9C7#1B`CWQDCs?wRPA$&KH7R%$gYD8>IoeB}lbj3lTno2~tW3=d8;~f3=d#ob z6dl@>Yw}A}Qk`Ag`GVAkp=-AI_{z9VNiL1-IY-52f!+|i(VTNZ{*vY6kP5n`!r7He zSr-Y{hB|YVn;C_BE{>1I;yCxcGrxyLHfJ%fr;n-JcOd&R=nTE{b~M?XZ0I(tk;_f? zo2IjIm)c}AlRW4jZh{(mB{?JQM!O{%H?Ay;fC5^uKA*?Uy-RoI{!qW`e|XdhJ;goj z0=XPC^%V|R55)`LvGJ;B+Zia5VI%6xAK;bi`g`gIml~p zFz0#Fv|cmH<7i)>thzK+BB>IRF-E5L^Rk*f9@iCbg2zd%Wo!D0Zd(2hI&>ys6aXFk z^DFA6tfo<9cOp9;`MtZvk=HIZ%2Mrz?68pc1gWjj{25qa&1Le#uJKhxGDSIDeDmbx z@oemffl{S#@Ju{g=V@_)N=20{_e3p&V-18L9G5TJ?2%ogPSNq>#wEWhSFCEEmsuJz zb*@`HJ}R&IX6>*LbF};j}!1}Itz6M94Cd+ZJ^kNEG+3zF&pwprQ%{z`Ov%M4s&qa{0rCiPBE?RRiA<@I4gsb)#)~B&j8Xb{M|5d`#;3#|47vT z)$b~-P~+$ynoMx~TOa4(0j4A6^RDFsPzV3dW&W3cU`R#^=vV{jdEOf7U|;ZU)j< zM56w!kN@|=|Eq`a-wXdQx7+_mcl@gt#NOciZAa3LAQY1`;LjY^vNw{g0_wdFFU-wl z{?+ThBRdQ1|Ce1&z;k)k*EhOBW@9i7a}Hp(&KqK+&zt`753nq|k65Q}4?z2^ z;aJr#a+?q;BYh#qiE_K>1mC9flU?m?Xjr2_g00;5{DyYf-48}WPE!Ww!Y(s+)*&IR z;>a7QohWFco`I<3x*Kt%A6R?Vet;66?8Y|_{jy^|!M{X8bIAaDZyQeo-0(>b>naQN4QE;cPKHjB6%UQ4y%li)y6q1UziUEo;_2UUTAu$A-ks)r9A6JY#;RlgJ9Q(f0#{gJF zA|W~J5KZ+1G^pf|MBZe0RW3bhSh&D+u-u%KUCLg`N{@u*C6In%uz?!|Uc&n7sH-g<+9ocCx%k(%7+7%AMhn6yB% z#1HV*w_DV7z2}0wpXe;(p5pulZd$){Y?k ztk!Rbti#NWy_Mmcy*#Gq*3eScFkT?9wi}k8l~@IR_?u#bl7x>G)?#P5o8|%}=mU^( z+F8dg1MKC362t2qJgpRnXN9?ekE`PJK-_(5)EoA`F_dxk{jNFvP9w^oS=FB+?Igy2QB&6Zt|a-3}rLbK|>!!~&{WA;v~H`zf@ zU&}FJBj!M{Yg+3c-1*sXI$ss@25NTlQWQY2V_~M%Lwv>8l`J2So5Qy_?vNK4A!xMQ&fk!CFt}M%W zroj?NfyUdom+%;H#$7!xpE85cyUDBlp96cgOz1vzNhkANT-@eKK6r1yMOc`Tw*9Lc zHZU(JF5d>WA%iwtvU3yFe#ZBdvm>)9OB#9>A<)Uc>Q!=cC5V~zUPtUTX0NH%;3@~e z2}8PR6;61yV5h@trnhQ9oU3#dxPMk6$c4L0MET0_h0Q7wp#JeAL^h~(4T$2bH{YZR zExi4Dp_*a`JYRH?`sBL_vZ3r-i0ZlKwo;RFoE3DN##C&OFBA9I@Rs*#Aydq~=Z^M{ z!pP>0Bgl>Wq9SZ@owt64sl=$S?ZA zGVDZPHGZ4|&47|(LOq}52eF;A{`_ec`5;*u(I1 zT3xg2);DD;?wWuSe3YJ;b24W>lTjA}QkEFM#!VgCM095fj8D9b#je?(s!?_K7~uL} z^u=4i+7q$~;Z9_aBmui)G zch~fFp8I5DTw@9n`l&x;m4yB>Abk93vH)rx$cLn!TWT;}Als+NAoiZk&Yvs}TY3bC zQl{f<%QxHgenU(0q?3^hXCd?gTP`2T0OUl0uD~mVmjLysFRhdhgYSa=oYS6l^>>zr z59IG>0N6VY63>{xpZuy0#Hkc!@?)KpWa znq+6JB#G&pK9*veHX>ol90Au(%HE)3jzXuxQeNjUUNsOdw|;WCi@uwV=OCP5P#U>+ z;VODJY}PJ-160YQKHCctpu(=8Sco+*#9~2m)W{}khwZnpc*gCqRqHGwGHr+CT(PVP zezVO3mIrS}DgklVny}dw@$!g&JSc=od zx(kU_8Wf$!M`=Vp@D2&UxqpdVOM`}?t+<3dexPwywn!c?%;rmMrGrhr!CbHWZp$BR zEP+v8N*U20`lx6M)#j)txcA?7dg&g8#iMt-Q5;Ht%KM>aM0NO@YMYz&lGapiRc!M?KrwW1lq;y52|FjB6#wy`Z*8t&<6&~$zj(~R7Ln_CD=P<5h}#v8 zVPa1qZXUB?rr%xER9l;FG#bzdP-G27X`34an$2d1H)K63>*8egy& ze;03-c4yc0^{)*se3UKeK$1KC97mpf!L16C;7($G5}jy%Uu{S^3LHqi&Pu3A!funG_-3eUaxr51f_~zDDRkO6JYl7HvA$A z60;4j_R|Ua&B~-Lqw$uFt4B{DD)y+@yW;6zyy@U_<)zKnnnL`2#|*rC^eY+`B_{su zYM$iMU*|bMmNAidjs6SB0_g|&JsWlW(tal26q1~hp@t`Oi~s%{ws|G?y;&#?o_XpE z68r5>-oM#v(b|Fkw49WkL$0Vf&VJ{GW~fRXCEfVPRB;THeU;{lJx4NIV6Ds- z?m)!yi!x|@)uI$rp6hRzrV3giLlY!uGYNzaqg)kKR?3s8e5E-QWES$`Pm3uq2^@Ke zlYlwq)Rq7zI~8J56-`+uIw+si<0^A% zD?a+CRpwSD_$TTheq0dn-=n6>&*pw>{Eq>(-DW{OX_q~lV&*|vv_d)`d#vR?2_7M_ z0HVoybOa8S!@#ewnCFAF#$k2_ z+Ow9g-`3lL$74apPtf-auNbdGIi$K}wOzgdOiOH!xsznpFWnb>5}Nm!BhADqNwen@ zUwr?mNf<^C`ro;JJ_oIt4AcI1-@E4Nv%PT-`;aY~dQu{0fsKx!^EAOn7W{0=z&S~M;xc@0Ll-XQf#C%80j&Jcx2FM%j z6@P*;b?-yjV7io0E@kMVyYZ7P{0Zz2I^Y^(#`Ksd9md}gf3$8lZp{%Q%ktU;mgC*(6-=!q|4$zy<&H;h8``FGemyt_dx_^%l*mo#$ujwN zI<+E{=s>z4dwHRdB<72`OQc;~L3(j!5Vl_7HpTNyDVma`XAfAI?S&lS2S(-rUMBVm zW6d-3ayaKQ@4s4AptbM+jKV-0W&%8JTSf&Jv#&eQ#+K zZ~yFjPnPBHhu}1s;)oj8MAi?_;_5#V_byUqUmMAp^I>+bZot530ToO^S)<$~22CVp%X zJ66of1)I8hUGX}=z*ar~ZKS8EsjdEdI(;(RugN!iGW+M7S8sfe_b2r|wX&itSxM{! zgCLE&wmyBe%8l;nPevmn{IC#_i>$% zp@H)=g#fF8Zth)J@=owZXz3k58JeE%qL6bk{x%@F`XSUoIL-s;M3Qky^I7+-VuYXe ze1wCKU=dO+8gktN*mBkoFPYhh7YT6?J{rRCxA`4FGjxBUM_L$)ZZ;dRIc2PCI7xn8 zJ&th>i7wnrHmtJNM5o@OM`*cZhAK+>?2^N$9>F2m}X?foON zg`BU$-G2h^iRV2r5z=t{yxYbX>?1aVtFcxJiMbq6nmU3`OsGbmUBNt;tT<6x*t#oj zAq!L-p276}aGzu)A!F_`dSZ&Dp*HN;h7FatuOacPT2{sK+|am86)eH&o^$ zT3*MN7`DwrpZayEi9lq6q1X?+EQUAhV89t?ab^zu&k(clkI$NAnOtEUs8EBK;lePC zXG=?hlGk4%_VxI|F`x&}Iw`H93tl6=$UG97UEK)N;XL&)Z`~E5VsEqJ)UqKCswG*_MezfNpshxE>nlU?0S% zUyibcE!c%0_X4sydAgSa51}um+vcY`j3;3khLN68@C)a9d@D-HIlRxjmJ0?*NnJX`#|L5@q1~l=cgG*Qd zy?Wu@vA~pFz9NvD?~d(JwsDl{s^ysXw%}#``2qqsc)O1wKt;J@R2cSLD$_?4#7+)B z#k)@V8hV{!mPCz&1$Q&MuP_#uGX0{<9$YPoO@BV|z5paa{I*T9A@_|`UjG6q-iEEG zPDF#F{?WRhq^mpv^2?RrE(uH+TwH!XZe*$mN{z?$Ds3+FKS^H0Yr3C#Ynw9tT!{M3 zS~zwz2U>^eH37`7g?;Ef{H-NFx}O(8mUQO|yxu5?)T!?B7GYgH;qAI~0G$>&li={C zS?zZpuYQK|pN|1(fyIs2OTD6Xj=w(XnePMp*qJFK;DUe>3Nl$wqLB0z_{ z<}Y-Ws93whXszj9jawV(OdYZq&rk~FlN*_|oN`OPFuv-$k|%s$gd~ ztp-?(?N+=K(Aj}IAHMoqZ#<3_TPk*RO*iGf9ANxE~!BNR$YFpLY#z`XX*id7(Z9?1|S~ZuWLS6dS+a-JOO-$-KpzH z{zE^D(TH+Wlh612S)QMZUY#s11c))`I^MGiiHNbJpkX8@J3;`Um_hNiGH=L9syYzC zRP-BoJE4*CsiT>SCODhm7PfN5)tR_%#e^T0xh`Fg4 z%`T>FbyBr6u=sPSVw@b+k#3oQUt;#lR%J zxRC6(JjD8nW!@kusei}gt)8*UjPu9uD|f+$Vlg5^NrhE2ZChk4NsZMoA^L7|C}Jgf zrKW>}ip;@O6+gUMt03JWol=5qLb?&8J4F#`*mMgbqJSVFNC*Vh9=@j80fiR^M1l2P!cXx>rR(ez{&h8Nh5X0UK;)#Q%f0gUnE=1`L^aNzZknBo2W*JonRt;C@%^FZFuKdia~N0BCjoQFuB%G8Aj>$!Avz@0AAL({Fu;*+}a zwxZ_m;kg?C^|=;g#I%mE8K#^F%ii2z=y@)-b)({-|7In%AMx1|S5P=liD+>69QvNc z;#d4Qt6TfgAsk$6P4vMqVso`MAhT?w_|{`{mB5LXH{$?-(pfz9*y!XF-@SpOU{K-9 z&)U$~-Txr8eV=S#%b4jSXn=)I)#@6Z2%pzibYNN1ZH>Z{Z!#t^b5}Qe0hyV;$7B;1 zInwUe_eN+dQNPz=&*G^njp${kk3$PGF4?lb(Gml>s_@djz(8UdJn}UNOW1-Gv`Pot{D;212$;WW88$v!h~!K zpzoM(tPHL;UK^Fv#D}V8Ep!g;9B!QypVWylWjGlNJRaU+$<(RvBS;nM!&%cis=}3- z53G7rRFAn}K6;h0;06q@n5-xl$Z9ES`m_Y%Nrzf`*JjGpTe-#fR;27)O<{trCQ-?^ zbn!({;ma&hR3%wfWTvb*O7TMd+o=t)jnXxQv38=z(~&rEU!|swN^O4GiR}=E*;z%} z$#KDf`cZ!Hs;c!E+tqjLcZOW7v{Fr7RC(T1Tw9oFKf2u^q1v(b=2hQk1J5mJm2TZ$ zpb7v0*y`K$60c$IL1&9I;$V6f(`FL3t!dEX8Na72`cP${m7Fw{r9SQ7m^F-7jvr8| z+5lq>Coi^k*hNKS(5rIp_=K?-hvGO5vElr^U)4~N@*s~x7Pol0N!)m0>*iKlYJ$YP z-clG7Et<_hP43m#;AcuQ#Y5JmkJjX?(w^YV7y9ZKZD@*+V772nG4paSWnLA=5B!+Y z3;&{f65oTmu52_Cp5SYH9cotD@ye2s5BC@?21`@%?z2(T56kU8!bvB9Lu-zXNq9nw z=E)@(UrLbxtp0k0Cxohqa&s<^b6a{lcypYcza)$`@@?ho;eJm34;8(>nXWaa*}j{A zkkBsV-5nL{zn_?c(%L1S-_jLQtf${4&3?_vWqii)o-R}|DtEI1nIbLT%Rk(T%AvA z!aw?%!bEs`E~U5OLygrqRR+P6t}=tV4i~2CcnZ$_Z`)x%-v3$wx`K`mt&aAdbB;BB4f%6R$8~S za^bP#_cQ^_Lsv*@;~wZM!`R3>qHNgFIBTyp^0W5;`$oF+_A|Y!k2_9%2T`)T;-U8j z&(Rv8?VJ*dLS`|<_~REaApNp<{?!}OdoMj&d&x-))zjrYtrV`P?fIVjfyNb0PmHTV z_cpl28$cIK^n@^cvJqGlR;#hPa)rSxvvhI zbX-+q<~)8^YvZ&V#OK8P+!sH*d0V=E`Bw@9jt6S}kUs@_HtyTila6)yTwaS_CWNu; zujaO_lrhWYtvQ2gt|AKZ92IN7lFb-@&A=!7qV)C0jl%0!&#-8V{)lGP?gUm`A~726 z%pnq*`uMQm^(KGSTSZk{lu zgzbz*5&5_8vc4wfrBC|}an%!!o#?)zS@^<6@&8z47q-dH=Deg`zpv8&B|rM5yhlNF zGIS+gvIma)=)3W|Raam0z?CLW3MW_<+hedGj=_Jq6UB9fKjKse)2kBGE%@W^fv7pvB+KR>D zUfe40=JXtTK|pz6V445va{Ky>gjegIS38S16y3FYlr@TB??u$ zSw9=VBd)bVu4{;{^ntgzvo2afBXyJ|S?_g|9>ER5R2?lLLNc9+yn04WFYLgOXN95d z@W@H{w5P>wy05jJ1c32@E#P$!=GlgN;JX_#wRggw;#Vr<=q~CO5XHvO`^Dtl7ddl% zN#|+%$o63-YI;p&e+rM@`|C7H5q3HSJ~2_xmm(da*dBCEc`mwUH@$=YaS3OPkHuK= zt|FbsLs|VQ!d>lxlMWdy3i_Y^XlSB(nDX`*x5T- zMBe5Z6Ysef=T9;ky0^zd-k7|KgC&psAta6%f?&ysVpZ2uSn7EKu(YctVvif3y(@pb z;eleRspa{al&?-jZ{o9*R6f~ueyK(Bbna^Y5e%z2eq%zY85vI)EAQ_9p^?m#^yPq# zw=d#DAey$i%GFF!!8p-L;AQzLyf^>_O?X>qeO?r=%CfO3kr1yW(pK^p<4ytUJ)}Qz zURN5bc%2{kz)2I*CA*2WBu}rqW&nSFP6B=IlL@SWw9Us=g1-C6jTX(!4 zH-i3YKOfIh&kAg${#dKbTW?M+<;j?PS!|g$Qs#-*{CS}^&hcer(@*n??-~+3h-C^( z;0X{=`4hwr8^&IXOLPKS!c~*+2pVz;hc(N6XfdEUyk5n%_R`i7E{%g`zOA|Dfg z*fv@EkV3cnn{xF7w96luL9JDbmq{3#tD${_%Zn^LSHmB3MoP3Iw3hDM7Roet!EKlr zTAsDJK<~FsvQTM|Z0XIxG}2fCUqSxLDh>S>r0fjU9f!872~Ah>2Tg2`MRWw5Cs!R@n!F2AnW?@&nlQ$vH=5eT^j3NaKHGL!oV-Upp)KJgt z1$ptVTL;6-MNjhapN-EQ%h~J&c7Wghk)vwnd|9o*%RWqt%)rUG!4rmW(3El!UMIXa zoQqFA)d!h1W!i*i_ef(tK3=PY(#a1goz$O_^Qc?v7=6MlGA+TpjJ6oBPYGLxfLx^( ze`a-U1!2Kt+4)U8I=7hh+a=7{jG9?er5&Y?;NZrT7Amxzg7~fEiclU-QG|1=+ioOJ zppslpRH3^H4z4&>P)u>*^QVnV?O;5aWJ_LsA~jS*5qXEL=Y0YmEi!&0bUkaGi9MIX z(79>I`Op9DVmYm`u~hLLHMd^)>%}Y4?+E~dfaR9ujT=y8c|({E*OY6gfI_IEqPZ!p zmG}CP+X4BN5NUmwbl#9`ANcwWdHR2l_ZfKrvgPIWGxh~9z%8TqXibNOk$V9D z&lc(N!=y#Ae?F4daNWMkHFUyHF`6}VBmQpO>B#$g5DGfgH_KDPd7oE2nPkX=By0h> zPL6ill~x0;M`r8z1}Rlh?r9eshn<;0)RIF)%<}ByU(S|1jlBPHhkuTNZFvl=t0C%L zw7_>go|9NaAfhIBs#(MUBhb~m^SV6ORCiLG~<{bVGLhqo^76K z&|Hl)kPE|;G{iu5&UjoFR+^-q<<@hspyG73VLgE7)->9!Jz<6SNW+x`}Ezq2jx9fU%yF)1KMNHUcoT8#Q zEjbumVN8>TP_}b%v0f|FAzb2WCVi*Ty2V`GpZYCWS+lMMYhiz3Y^;6ajfg&IED3j6 zEhTrfezLYes_&Sl7dc*S+;Ng34(_F3crJK+Qe>(qT@8oS5Ao>S;j&2=cY{=G~OLL&TWBHWQ)vLF$`z#&FYu>~(eSCY&84FWG>|Px-hhJeJr6)swvL*D$V4r1Y^8{oK0{l|4dMam;Pmwx3+k;n}^zsY=6H1hVnyzV>pk)DobXrP+#B)<`!zZ6cN&Eg?Z?$SW zb}UxgIP&%1qbdfFR>huBMOElg zqrB-?1pfL2C@ssD!@Itg3wyGX8oZs%YW*Lz1NN7c#}7Y!H(+0-*$o?=45U8j)mbya zv4ksouQ+lJi8n1jiL8+xf`N8ADqjy3wspmM%l2VVEq_w`!tCg)C4H4(V?J@-RQG^O z)8MGOs;dy%{#KGK3xkD?MBAOwSerDXNX-tt@yc<{3Nl?)b^|{;S=jEOZfLqw@ugxsrp3+7~Fy$_RH!%Qg49~iLvuKOWIpPS10zw>Nhf zhGfK0v?mOSvWrjcuf}{VxvP(6S{rBwBwWmn>4}vXISbS+)=0Gtx~~zSGXNgVphz&=nUU_1{^ZKOYyi(oTmC!}6ceO)6z*jI^7+i^(<97}O!eXVL9?Hm&$8wNjM)ey`qG0h_F zA)9Lb!yEXvCK`l5hT^a7BJ(Jkx9e7?pAKu}yPFc7LBJddj81Nn1qg^Yn?iY9eNrF+ z5g0FeEr^AIax!4;h;_G(YHptS=6Rj9{_$@eoMVAtgcLz5>hrP}?S&+I);7U`$WF@N zSef{Za7ar$F;>8Acxck3wb}@1elNs6sLVExxIGUu@_22Rn1w9=8nnkOKy^tma>a@i zi^P_7;PSr*B1oQO7km8MOr<HyYOzSkequxzOLdwlLtgGQ z3B*vwVu2y7vySK{J>X(atY-CG5%rUB;lCk+_`JdmO591_QqkspVO%I}dQSNUI@8I# zA%Z%-aiKo=;RDLCNp$01gk0pC1ff{UNbKAfjw5zU{Eb7nwiYRl;6++M#>Oqv7ngAO z+3ej}gM$X7gwiqIOC6Lx%a-6l{f&qME`?D~YSv3MMUjjH{rIMuZhxIX0=g^xW$$;E zxFWmmAm}0~$OK=$BVJb!a&~tW#Zfnv5AXDnA>V{$^W6GTUk~_lp2zFg#!=zI zhDrIrLrU1;k)8Xz9}eiLD;48v$z#gR!@q`bf1d^wXjpcafcLuV__?Ml!iDmKGw>wQ z`r-*mqSNige#a}8-urfhQConzt%{svw9Ek2k?$q3_AZZhk3!TlokSQX;n1S{xi(G7i$H@rG1YxnsJM5JqUF~wDEIGEss*0V_%BFM z#Al+PS6P2zH)@A!DdD4@`=I=AN=9Hsf7Ze_lq*6;hZ?5qskX1f<4-w~xb8Qgz2NoW z>@NAmI6#iA%DU$jJFm@x%trGZK+L zOw5KDfcT%xx}XFoU*K8%F8^!Hi`6${{(-qM8#Uw9++^-<8AOEgqzL#I3M2@$S$IiRRIz*vU8npBRcoS=RDP;;jSQaF7J|E#OsSr_-ntT)hRyQuXP_PK38S14Uz-@p3F{qa%bPYR6W~yFZmPt7m&Nw&!-;i z;1SiN?1o8|lDz{J*5-oc^j6?uomR12nY8E5ZMU$shd1ui*zs~-ao+% zAq60jI8&4#?0U~#S0mR>7U-GTdI`9+lc7wlc@tS(=uHLG2K}mr8J3GB=SV@-L9Mdm2LiHZKzE*xHG=hQZIucDvHQ~qE{>B4b5)3XD>uj3yvJ{+FHZ)s7bhmkdJxm#uKNdEE&&}qR}+N z!gxLEMe$X%Pcd(o%KpU*cQ-6w<)CDXwY)^d4&AUu<&Sq7IsKL%~vC|&-o|vs_hm$roW6vbxDY^ zUP)!`zX$VOM;F!dDAcQJjcG))Xr0SyRINCdD9F=2dvg9UcTVJo8;ssZg6{$R5e&KzlA}rh79W z%vo#EIU;ODB#AWY146YAa9b$N8f`C_NuI%dK*{o(kGk)iB2Z?Iaq)(krKZXsH9gcC z9lx#SDHK7ikhfG!?%)$abLPsP2IyjB>+@?&I0=XQl3Dl((n0t<7!4JdS@?>CL22>s z%Q=*Hldq|Se!2`PvwcG--3u0kuR?~Ove6fF&Z%K}C34m8(V4`kzT2sP$H!jGd{SET z=d_aR{&vMg0=I;-8@=-g(Pf$Nrd}a&%1<_c%d2FI7`v)4a=*s^IrjBy1c4{&jGSmU zHOi*6!tQO_+kw5|-xsONn{a8yywJS*etyLB;~zL=0xiGzmE7~%mwmH)C3)j-lbD8* zr6uv!Z8Ad$$u)T%l|wIcLJRs@6zZnQ?LC-=Svx662Qm#^cCuX=_VOV-C!@7Cz4VmQ z_vUWcvUd*^`^sLsLGH(a@yOD>4tkU+lde8X*JJ1NtD3OMfS4f}eDQwgZ{vYmBQXCw z7o=t?h!xrPxo1}|D@O9-J@;&!E$)yL6dNDtBAI@FtE^opbP^JYtgK~cL~6SJ-ljBk zmi~{>wvCoDTBM9AjAEg#BR$AYiU{dAT)JYR`PzuPp@ll}RKDo#_-vSOTtCLv$AW2J z36)#t$D_~_iL-cXLFxXB%QnzkO#CtEFYYiUqm82 z#xk2SYk(kjpnom-li4IWb<(B$sfF<(ZX-SIj0+BnL5$b7@oWA!oV(1wvv1=lTu(*)DWJ-5fhXv)q#ch6E0=6e|vd8OCp%(Au5D!pJ_SR zyD?3}JoZ=INccVL>69bC<(#nc=O3Qi3aL}L$Sqf}(4sw4O}d9$%GbWv%-p=$V(~JY zjGZ7&ulb+Q;L8@*+QeADQ@{Hty>6QqO;OQ0&az8#o)}C;zgo;Y|2#MTV-{gps+?yL zwf^napOKvz%B*Rvca)|s$cN8nu81ZquF8F;VO^eXcu`zv`3yn1s&J9Vkneyz4rJz% z1i``1rn}Rhn2f4VqMQ0ZBtE;4dl$n*rmdf`K-$&)`(-%$@!TO^u$-S!ha3)-_)qK` z!Y-0jEmvcGwjZ3=D60Tnknw{yG-+^7huv@)rX|R26z> z-CwOi5>-alRZr9LmLvaGsS(wU2QnaWv1GqMb(2IUu2!TUN79VdSr30lVdWkbd&(n6 z5%wE#9`bF{yL{s>RF}WV3f>IzrbJ&Q?EABwI&cU%tpWZ9UPAC-O2%wxl`!6p&rfPVDqsS&Zxp z<$3l*JrPR5iTh8z(nM(Y@nyd7-T3w=tcHD`XGU@+DAQb)^vZ?$A03pmmm}I70t19U zGnvG(^lWjfxB5ivu#|`LP(Ep5xLos)8qr^R!g-RD znDi**vuH3lYdIP_gsB?MgGoo+GbWbAj}mwqe^)o2-L+Py)_?INckO^Y3uAG-wG2dP zrJGrYX93yUB@)*Nk$|?bp2*w3xv-fA>Gl*0`K1V`G>s!hRq0Dd$Z%v2v%lqmYgEtA z?(?&~D^vw9RF6o8bl1c|qQ$t7Pn|X@b&$m~^M>RV0zdJxN6Qwzx1z6jEyhcKhN5Wh zq0(pKyG?QT%l;60=TG&l{kXFLfUmc(`1Hxo0DX*(n&aRl(?Ju6(NVX6Tu7OzLPd;s z4!yC36;beUvTaz9H~RxBIYYU&gD{n8ftccMc?Tn!heHmbWX*hxK;na;2|nj+q)NDs z_oE}kC|dOme$k^rn|;l;GXL+8Bh(=rpo1HJ%A2r_)^|z5I5_&Np0&O2{cciYPEHIM z1>@Ju5lCmCNRy!0IC&CvigwA4>N$@^yBx&yEc#l~8a-cIvu9oIjH-~eWDFVixv}ir zgHtnVq56=kh^)#rR`hpS(m_!aKH)WNkD&sHQ=_NG>+g_D3hQZ3><|Vu){!`Gq&?$8 zs#)v@@s$65TJ?+osL{VRZzw=w8211AM`{E4@e{>2xc}dN{D1%U|A$4D`rj+_zgOn} z$ET@M>L<*f3P0a_^;ZrNr(Mq1k;8$P@PGM}PEZfD&mgF^hW({&{?EVU0Nn2P%bVEn zCNmE3U?W`PK0I1-;NH{PUn?qAK(J-_kND&(dU)?wHeX_wL%7!VO~g>D2xAIo^T1un znf|0|T-5Vb(CNaBL2V5QatV+DA0T4#NofpBm7}-|%H8 zCQm_ImFLA$6;!1POtIpw-M;f{&#d?XdmUKe~FB9nNJVcD14oFEMDs6Eq(U2Bh zYd<6sKoX3l`qb+L+{aHXz4|3}n1{)C#Rj6-4`r|t(ItXPig#fU& zj$$jJV4Uu^EvZ?KK!LE^79gd&Z-O~$pzYZ3AAz}3nyEh;|C9!uvDX4{lLs)}Sf33~ zkN_l(D>NRDmaYqb?=RXqO9AHL>~Or$2KlnNoAJ3ZX7T3`X&#;oD$gQ4Q+G@3>-kGS z)l?}|f-rpg%7ns=fc>iKB&<6ZL`e^`5k+!Y+x712Vrl#IpqGEP=>rK-3W&;4dj?2S z22s!p<^c-We8RI4LTt73* ziEKIuAyW?%1c2|Efjc6Jq{Bhf_fY){Mi|4F8`eazZ9^q3Jp{OgJUs2aP2gjxYSsV0@* z4ea%Ow7+q0BjRerVVHDA#p%ND1KYiLA>4injMX%V^>xrZ^6s+wzxWUCcvVEW&IUX=sFa z=6@a)z(F4EdK?uiY}W&ZW|EkCVCMbYVbGx*KpTD`;+BT*M*0a_kv8Gq>q7zC`h}ST z?t?NO@ixSNpxtY2O72v%lw5YSH5wRYW8=X3Dt-6&QpnE7>L<>Jw!dvz`z$O>9UfoW z5Xe5RIBwTEy|2X@RF%RV#6OWrPy^H0FK$-mTLcN z2JtMzF)F(qs}2=crTSM@+;44q6t7^VcOdMJl1B03Ndto-Fk>0t*-P|>2)oP|@eU9DFQUT1I;TF6?ad51{{^1(URo}m^n_d<;VZA?xlKIHad3JfTkZ)>?P|1StpJ>QG zJKtaGnMJq(2UdKGg>Tm(rFEFdCrX;{9>BK_S^Mq6fvwg$zNdhJ3N8*`g#-MZz9SuWiD^SOn}eJnRHj-{4VW9gq@?xIG{lw4)K>L>m0uv zjW-Et^)X)KR+L3`IW0N5QSU0@yia=O^i#E1X!21tQn#2mpq$#+vc1h6Sa%X!J7kZY zXi*8i(JtKY9Nj-7LYOihErWZQ&_G`H&mS7(A05SQ0r7NLfvJNlHUa^q)O(3U(uiFx zr$VPLDR=N$*jDbg730Q6mzBLA17zC9L$a*~20$1Ldt_lua9*E}(*0KW_hm7aBjuji zDAPBh%Gl0%(fjpJSX19^QNo)QiSf-UmpHF$jJ$F5iY-*URt(~=538J1$2}@^k0f!w z&$YZ1T5@3%oOIyJxSGEP*qv8_TP*9f7rjn0o2?&v?h_iT3Cm?10zli>{dNyF7CF8t z6`$Yl*2dE%u8q1E7rsu|m(tOcdY;fw58Z8909~4CwA<8hF;&;bMfIzF!gkGh%gb(* z+{T=UG*!}Cz~k-U;F18l*pquv*liXwS$*Q^j$T5OeYpldNDA3ma}NibJ&RNozrg=l`N zZKAfxL{G9rIbet|#Ma$0GIXrf@%N0s-tJxHrj;d%#n@7j^{(YkQKXV5?i3c4q>sb~ z+Q>XySzc)dFpo4tCq94UE5n$R7s*O9&Dv}HeK(}-5ET(x?0)`wiFv5+@tViF?+#b49}jD5(;*1dx?r8oo+6j~Plm$F$Whd!#I`Rk3#AiIm%!)yqI2ds zpS8*^!Wt42RHHGg7DoR_sK`n6k9M6Xnq0c5xLjrRg{%$dw;`78>1MKSje!yFMks@m-(1%^}_>pI4L$HVHd~CM8e6I$|2kGyxbe6_O5koU4sIirNO3e zS%iEH*W9|@wK%`ad-ZPAbB0MtvVB#N+Q({BvFcoB$P%_rd&1ULUj19Nh=cmkYgrKq z*4AxrKrZz0B8%}wR&h!1cC1$?kMu7}T&dG1peVj>BmYo|G2*4Y(6N=0pUj#LJqbnw zk$QUgI^yd;CHs;tW~iq3kGrb;wD^IfZ|!}oZ7sgVCB*#mC;Ls2L?SVdLS=FnwJ*R)m5 zWUt~*HCvA(RMzoaJf#|TO_<9~_emngBjM@D1+cQwPcSc$1!GC)s2@KpSP6~80(EnY z2bJuV1^lj;M;`_@#D;>dMRL}@OX6{$4q0cvU+nJAd%Ja~q4=IrDI1ipzm5z??f}Y8 zf)T-njqzYUcn}w5`766FnNYM$d|yElr8k-MKQ+V4{N1;V)axGmhpLIk${FJ) z9CEqM-J`$BG=_^?BXSXO`?AGW?7*UU$os}FW)s`9E;ejN;z^0i(_&MrGe)3F`DIOEmd~V446YRsPNS4&m{L2piogH5%Uc`r`r zT*b|qJ^mJD?Ur6`OkpEb=R@A1)Yf5;13ak`+76TYz8%EklSWO-9Cu__#}Fv8?p2WD zsx{9o@Y7tsV1pN zNLSj`YOTLa0LuRLd%K24ddnzo>_B|V1K(5N8C)k_!J*IGMbOm-1Jz$;RIgnWEWbh| zwy=hzs;)n7t0(Y5QPqAKDXgS?o}=BYcTq(z)4lo8_odv8+)fj<8mQ4A{Jal3)h*?hG4SbEbg#Tmd=Fgl8=Fl~_*dC2(3-XiHbSJStlBPE(*d0X_KEHkG zA4el=x6dE^zj{)+)-TT;WyUw52Lv`$is_O4@vNw2GC@;9mp1A>hBGgKmE(;;8XWD{-( z;61krYr0#xS?piDNH6Z~?D{@p4((FL)%fRp?&S?ukEH-*F%Rg*w2AbG*_+-Uf_B;- z(FVPf3iPY}qG657fE~Feg`9t(<5ncZ%y8VW)BwRwlN;w&!?XccLJ_Tqz28J4M zR;1&Qt=TVR2)fF$_Kks&pM*Y&*T4LLTE(ybWwCoL3_^<+eP12XB5zxlq=AErjjZN} z?UwMbotO6ukw~t}bO?4COYYo@gC3IOiJn5J`dSHf_3&*H%nzJXi5}~0pXHwo4KT(x zJX3n+bi*m=GUp=!rUl7NMymdrhu^lnp(GcmOxk|OU@-!$RYR`fgjv)(nT7>QGi~UE zeSWk~EkH`rw5AjJ1!>28`$mBVm=VrT^o)dOFGavNBxhY0|ZT8N{-}=UIVWs#OHjrFS+E;kg=whKY4395^ zQc1vOZHQV%^yl5kVh$C!WPOH=6APi^d2`FpHQHH3onYbfV*Gv9A5)Y;KT`$Dws!Ml z2`kTPWp%OM{2FwE`^g`BnqRyR2y3D(cg#II)v=TEpP*rK!VEl6q#JcUgFf5#v+@3O z>6{`l`+Om?r|W}q?`TnGFIC^=rqqgxEK}1)Oyt|c8FQKH^r|CoKzfIky6C`fR6VaM z4@czK)c|v1;WU%cSxf;ZK(Ym`;J>nCh{Ihk-vJ|_PWV46WIW3&Fw%=^>!Epb z1@jq<+>OC@ zKzGA;N)c_95cc}Y(##`V^A0o#`tufI*djSvzpd+xydZN!|0K`K>QrSNn>iXlhW4mV;0@C3zaT z|0vu*fz5zDSe`)YtVDV; zFSoO1IyZ;bA_@f0haW`ca#j#O*xL6g;_`3Q^e)S$X2T$}x3HpL;9s8CznDH_BKS+X z;$ugOzRp3+H`H9phk4qJQHrdh8%U{isQS#Hwfk=?nTFnl8%11 z=ljSMdhcj_n6L?z47;9>1~-xy|0c@Nng#WU^xiXN&-(F-T@q}>?OE}x*95ge82iYv zCjWP)BG0^RK#5a>%hRN&)~dhVCgZK8sC#_y>2dcHVRQ+l*`Y81v@P#u)5gPyTMj^uz@}Ed$ zBjUes@h*d-1RZ5j+5vEsPQ*oHy-F~l)qg;rehxsIP@+8jzqXhZDIua*xkuYe(wIBe zCa+$f!c#Q!50?JQn*6Kx@hD@Tr8nyU5b5XJS{u8oWR4m7v4V18YE(0g9P{kC`}&EcYLA2X|ue_pz~p1orL} z6>Gikj4H@aIKOIa=6$*UcvP(_$-Gg(+-5l$Di*vH1-ln_?FJF%TA+k>3}0 z+9Ne0VUDnJ2x}rQ z8(y-p?`n>@@bk~So0vnIwADU0TCg~3ez9!6-OTl?{`)g_O$9hMTF4!1>@liCw6&kb zsv@V;0V&2)Xn<8X#=xdrQ4(ztM1az3={^(Wu;y?X5{`XZ&cNS)E2e>s-#@b zD`HUV+%?A)^f&zc05@V?t-FDmiI?X=Y*h2kP5vVu3`l#{a|X!Pt$tZXiyfOi(aYWJ zb7O9mw>AI$*`!J}=Ee6Se|WDXNic8c(6@-r&#M_?@Kv?)Wo_m@{VG5-)o60g=8&K- z7_qsPtW4?6-z;-_%Nk=ZM4k7SvhT2CpQT9|&1`VAb+A@_*_LG#)o&&l@1iQ<@KE7g zp04hzwKvZ2Fa|REAJG>H1pj>#?2)t%Hs&P-w)crlV(qGBMPq78Oro@h{XDBD`axaC z8ZIn%Hu0!VC;rY4lmB`7xw*+nabzrbu}h;Bi_`m5dZa~azM(}52U@36Op?#pfntZ% zPgv=s58bq4x%LxE7VR-U!PlNgJhe;tz#0r*C+z5sXdDR9O}@M$3IxZuzmP!T7FmuK z{>{{(caePlD7_?#JG?}aR(AfhqCKCzrLe1fY_uhm=>D#*8QQTZ5b{)5o*v#RTX2c` z8PHDEdFOF6tCtvwLE7OBo0oS8#@A5p+S$>S-9rE-XHXes_3W@s`1|zyzSGuHX&#Xm zhV(Fg$U~#l4fCE`(0DwSJgBwHuPzadSkA(qNf9t0bbS^63XoI zXNzhWDJo~=)+88Nn(mwL|FfkLbgUiKk{0Q_V3{apE-w75xS~8lDDzi{p~GkCzF24{ zMbt*@NHF@?4Vzl<3u*46pKqu@>l*8^d?!6>D{&6(7X!f};n^Wm_KJ*Msf~x52xuUB zI2v}!ec%sF(u6}!OM@!6JL3loD+7b(w%(TU#C!~YzY zjpF;=wyU+Es{I<*f9LUxG~wT?Fz>-7SkS~{t`QiIk6i*}hclg5SO+6X%}^6dncI%;$~Y-e~hx<#pI6-sRNgU&2?iOdfg5rGqE5k%`J^(9P)!Vfb8XkB7Ve=mRq z=&@|EsL%Y3aojf@1eIj2JVn#;D+-mH1|xz-WWmPFQXN)oc!k&L|IO6ln7s0dX^w*j zwCU|3lpHEqyyA{fU@N~epw3(VbOZuAjWziKS$cN(U*#tS0jW-V4TB_yAo`!-@S{Ms zZV~uENU9T&)wqJ%`ydig1Q-+n2XxQs;GCY+lpA!0W8h){nE_!Q{9H=;EO&wrvO`V9k>BW$u4GyH)TfsV5PP#> z!g-pFQboLp&8M}Xrv=v@xF9OyurekWYHj=1kk-N&~4;2!=s_5#Q@Xcr1-bsrzcZJ4GpWl&p+uMY? z#YO2&IG9eyLt8sr0N53)j@IG&`DRUz!7q{#t@Gj0k$esWtNgeA4utCWccS-H zzaUqqr{@<9{iA5~7MzK364H`UA@R_VKRZ4H)=1shKw`Q}F#dC~m7}7EGzegafFk~! zeF)XD-0IAhT^IWPxS1xIz#q5QZu~{LKs&8lYacPjZ%G~Wute4SOOzai#Ris4&h4=a zU@9EZtPVslhpew*+~PxzWgN{OE>0tPTJsQx#j#3Q^cJd!pHwwRu*uu(oCYD9iLbMp z(nO=U_OZ(ga+K1d>Vw3nP#Ry!fzB`vY00;*0*}4_Y|s5?{lQ!gCOLAn`E&x9b9wHo zofxi5`9WZ5XPtBvmcY&02D@`dp`geQ^_tX&26NqELq_cU5A@}8Z18hhJ|8x&6@CLz zs>p5VKqn7s`TqbrfaT{69)&& zAe9o`N@&=d1xo=%%g9%pHH-0~zjRP{N#kcrACb8Dm!B~=>( zgnsAj*mY@54=dt1cYmzi)4hJ1Y1RXxSe<0clcb|&Y8HOxqfkZMMypOCaZX5=vKse5 zlB=G6mAQO(Vuh(Sa}mf&}@4HS@;6K;Dpi~qg=jE>74KtTKd`q)(1?#Xgc;PEK(qk;0A&ZIo*_r5N2+ph@=g~*1`D7x0e zolkR4h=iBHiXYgiZe2YBPou!dhZ|oI1^Y};XcbQR^};v%@vGu3`A**c z5KUP$#n{Q-V)0DIL@uWQ|6|RU(pGn0fwD%)1=yC~1yQ%cLS9I9s`%NJNoLF{qHgCW z?76a-C}C`LXS~94IM|8)Q}vn3U8g_Gt_SdthyIOlY`(13*``>=I|Wm@FO&Korr?ZioEXci-z++H;XvCL z7I&>i8uqNocGNijDq(n{1(3fyHhalu^VcS(A%tpFqo-v$O0A`@Yp+811eL_ag3!1T z&K4#M1qv+U@Mk=a>quQkUv#qhJJ>c(cm$0yL4pDTWwEV<>V5DA;I$3LE@?$^5Gl*P zyz(TjW;K95?bo3dZ)nGWTxx(W1)b*ZokYW}#e}&suWTK&`)62>5Yy$VEA~^1u$97D zC80m=9*E%nd}$H>g5VMe5VBdH1{2i;n>Wj8DoudS?K&)jvaAft%aN=k8}J@#Z2qug zDyRz*rF}dh8|BZ}%db7ex#(l7(!CnDEtTuR{t7?rYY@T6H;D2OKXrZwHK?WbcMz!x zqj@k^Iw*Ql`iWIn0N2#Cr+wG}0daWb2*miN3e`rvC#D{M-v*m$e!6dA6{QFO)RB+A zYk!?{Pe*!TUx`mra;JRP?`NCoKP0ojcg*$?kQkQZi)OmAWDK!c#?9yruH*RBMVoI- zZS8TflYD5ue$eI;Sy4@fz5(?KBR#0l#X$-14FT4b&^h9&6Np~K^P<&+F!94+-te#g z{7YtQyU7-BT2EePlaxKETNHz#PRrLP;?xn`XAzR{(e85P{mj24Utk#q&JzLT`v9R*Z&VE zvwQg?q6N!~ij<-s$(X4g*7|(_1w__odF?F1<0*->uWb8|MazORx+~x{E#O)?KakZ{ z1yw@Lk-IcNQXbe}Oud;^QxX`w%iazH2$^okW?~ycKr4M3kci)MBVI4M_g*bLuAILO zzd$6Hh4?OVa}<(c4(hUg$XH%HF=jHM2jZ;Wp=Jdj-A*}$Pxeb1X@=|M(*l&~KzPNH zA<6CWhk;;IAly{*Eec1ZB%d8n3Nq->Ku|e+c+=YxDA|G+^GS*T*ObD0x0_S1eK zH?+bjXhQ1BWdPZ z&ck_g7H?Fu=Ff>w0JK_Lb&9VXsOPPN!yzF}kqvN~V}+c}P{rB3S2_zmeilD7e=ha_ zA54t%!dbk8usx1|x4G%HBmxXvt!$fA2NKf-0CjQP7HYyFbLTfuKc0bbNsUdl z$s*AereDsy5b&V9AxO7+JlyEgv2+i_DEh}*!%mq-E)>r-l+3>^>F64f$LsjWnMx7A zdw%aC2G%rxiIVpw^U)*Uo3V)8<*j)j)p4AUlrGlfjherBl`4}DR!zXuchb4H9x_M5 z|Hz?Co)M=CL=UD0>k|AB%Ee+;Al` zCON{{3HAFVItYxZ;v_Node#mIp{@xYW;it{qyD)`wkMMtxZQ;3nRJVAd*EvHlPxjt zkSn~GX^JDE4^+cJJn9>tewfc}K&i>86mx9+j*xJVl()Zg5}_}-9<}&Hg_ZQc<2K`b z_g6dIAyj+>0H;krt_kB6Bjq#?jVu7x)pKaJJgF`ADGqMS8Y2M?oZU0-XJG1#YNxiL zrSasxkGB?LxPEc^6G5Un(6~%e@3abRKJJPzBMDc^)S29MX+va(5MF|@1HqVlW;M}+ zS}r8;j+S(E8HlJ@0<8Pj!sA*QFEC#F)H2Al7}y0ju~g2L<7=6s0zx5yFG>WE>Kl<5 zJKOrnXo`ubB~}!NkUj6NfapgXD?3p|W^T&6&qq@bp^!Fx7A64cAkXX09YEDjsJ-Cke~jf$vAv603Zz3GYTvZ=dBEq}$l<@$vkI_its{ z$~FA>4+r4yHAknH&jY7b_sLmsBP&^)th`YA6JlMr@>&tk%Xcfpf`xx`VAV_P;s3$j zTLxv>uj|8#sGuMzAq~Mb(k0SJ-!w=|D3a2Rl)UHV zv-UfC&-~ZyHS>OXKm2F-@QjP4_jTRBIM3re!mrBr2KgIp<<*rCJ2QF?b~nk}?ou8- z%i+DSQBI_PApMDJEdF~f#nLyQ1|K;_vnldhl{Psd_OVx3bu~#xkIX!!qs+dhMn`5~ z%N1rWnGU%CrAgmh()#Yz&Pfn%j%+4;x`;IzFKt5m_e?p4_0N5262`(WXgFOSfS;^3 zJyh2Rlij5#1*cqbK69m(stgWfprug9KgG7E@=rg%?9lBC%k|BEC+<4_70V3VBAC!f zlcm7^$#%PQxP}VF`T)ItS#dWQ5T^{C!AB=u=&4z9unK=KEpNtP^u9P15;L=9V5XgP z^DHy`wtmt>a zAAf&6G2`HeQaovIm4f?SUe@<}s z?wk#d&b@TXd9ctaNyy$96NL=FG9m;2w2=4oP`;8 zN-s0!RXY+hv}7Q727|xgl1BLlxSjbciA-1EjIdwz?B!hOjB)cZ(TEGdS=jyqPJgVO zHbo^Lo}c8`_YR&Zva1DuBZ8iGKZ+&a-}7ZPsc=izkWNz@fK`>lhM1JPJbcu!YosYSCMXs++l4?%Ny^r$t?(5L3gEkP2;_lpkk5LA}ak_ zuxfFgpE^emTd#7fjbeh2VRlyIx}fAk8~lJdi%9&PVu2zdBUi@4NMqa2>wS(G95QvXnfjFXa3@|D3> z3pXdgYVPVDJuBh8z>>L3D9+sUk+^OK1}`8AJ^BPQ#R(sTo_PWddUhe-pPQY5Md6N7 zrCvpCHf%Y%NfMMA28EyH+m+lT!~kZ+N6LUr z*s@?up}mC=UT1%+#c}yn4@aU#J0&xCF1Lj#&QZAd%0wQZITy}#7`Pc1edwvy7K2?o zvXO}SDJ1$mf@q?+aikbxB_DV-{`d^{=@{1o95O3Gxmw5rPpu#vGhwb~*m0KG&lnwA)d?+%vQXW=^J$hM*_2#w`K)fD&weRF6 zZ<8K_v-8pMn@^7mYFJHqkp+<8zXYV;0ad&_md6aEMnYrJ6jn%8UJE4DyL0Wo6@TMu zI2MWw$MAn%{E2 zo<%MAe9qam;QaNim7Coe4^y@772Fio!e&e+y&co<@A<>I2s}jS?mZl-Qyanht~d^? zX-=0Q0@*e&%(%H%wh@DDn(x?~&je=7`>CSw0OX7E#q+Id^pNx+$R(#b=NrreP55K` z3)Ij=GKmzR$!8E-^wbYaj9#G%9G7gDo@q7BtJ26yljazb`QPD0rB)Vt| zVwxrCj63XmY9c6)kiUDwMEQr@lgw^pj_F2BSwtqbz{qsqc%^xeig`41NTF(sb?Vax zv1%qbx#WNJOW}QT4jA@cC4C!TSo!lx@=F%NB?(Jxk)BZreZ84>Lqc{xUG7Tg%9sc2 z4C~4dVE9mAq#W`ff%*(4NM&1KC_T`BO^0t6Eo>}E@F$vuD8tQSJ}*}+XCnPOeg8jj z?$9JYdpkGI?cy&O+~J<3gw+JSo9Js5a-CgoZg=@K7#MSurckYq zXM}KE$feyPGW1l=aOEaf;TJ=N`uy%yq$H-qId_|2!VloJvZf0cXqrr5q*O+(K#JQ5~sHfb!+}0_;^Ul|ijd^r|JrI3=0v+{+WG&}bOYI}c zv>neaQm@TBs+K=dq4!`M8txlu_{@-BCQkOOUgLp4b{*%zJKRWeb_zo?%L{8?YA(!T zdHGtAad7W`Wk$B|8Q~Z2(0;C3hd?sR?GxBM#TnMRmP^_ccT6LBwt>NpnPHB{bqU2H z4v4;-D!Eu}gAV!j3a}UUI4?hYbTKaWJwG=W+F~=j1Y?8Sg2%VHtD-xcVHyGyf1XVN62hC)L84&o1`+G<=&lJi&8| zm9o^^nc;Ck*IRyJPX#)AwTU)Z;&={-Bxp)f;}-^B4?&k?zt->UJh#heA;}&ZS`41( zkE3!^hNtwR|B?ORo< zJt!h>j?xX@QXKLCxJkl=Wd>XgDfr#hZ@Vjc>`fGHz-Mg59Fu(^K)z6kg&jsjY|ctA zoSw&3Z^pX^C4r9Nz*!*=b8AAJ7hQ^3m+Pl|MI?MCkK*g?g{TBP=W04RznVjKSLU{7 zeu@rz1ev7SJ6zY4-oq|^uOchAu6B3hO??^y)-!-1YA(_@{%K2m5&r|BsCQG>ppG3s zSoWrQ*%y{@g{%$b-XuBYdKjG2X!?emZ%3QM*vvkBtJo#AwK8|80UITK&`lHY3QLpT zWZxkiRZ5h2O2I%R%ghtpg=X;|KseRpGnyZ7Tf$wil!n=@Sc@j3WSISQ0|;C9W45%T zx_@v$+5=kYt_;)NwA1N=X$nhb9wMWdM3X2b{xNjE)p~3#PhKHI*|PAjWT#7^C{ucU zF#cw^6Jy_SknI3XM-&27Ede@TTsp$?GRBD`6eT5Xaq0X1*qb9QZKxNgF}p1cTmgBI zwJH^nOihY5)4oJjG-y4jeD9NUH@|23B=#LT(nS1&80>JG#EwezY8e@Mdn7u4D3TAL zQgmQ=MT0@T{}mv^qVO@?ygnydamDJocU!02$QPBL=szD}u5Z!gBV!dFiRS0TbXhJp z%uimwfg?*U0|O{7qLG+T8zOS%Ahh(GP*i}8MM^hA*_pXT=BRYr77q{6&(n9GmxJNd zdyC9+UZ}la2nJeNJ$hAw0wHF{I5~%)i0my=IOvIZh9fT~*4Xlrx0j;%`K5vI{<&a+ z!!-<*C2LU;77o&MaOAW5WFNmmg;a;VW$O2#C;Wf4E&ujE(knn!pTCx+*8FdO<-h+m z=^h9nLa0yi7Wf~*nEHSD@&AS#P8CtIxKLsLFTaZrM5pP1*0dqa{z>yc_~_!8HunL{ z_$GRTSLr|c^`|cZtFGJ0A5#AhJ_{5uw}2*Kdu%<#^&kEE(*J$&|85EW-xvSyR{#G` zSA4M`5kHu0WkUOS=LlM_EF@h_z#ft$2f)kTU@<4}3B79Kxo5LT`f?`t&{R6>U0@;T5S5uJg(p%z zlj4UM$&A5Go7Hf3x-kHrC!{^VcBESAMGzK|Z8XCGkn=3Q7eQ;nnFOX7^wXaaQTPqZ zI5k1{4cFF(K3g)b;FB*0nI=BFaKOqULx;8F|B}Z2T5W7_()9#T=!qDV)dkkjHbpJ~ zBgPtu3phPNIF*K^YYCvF$%0Gx9->xy(Z!0G{gU}V=Zm-z$gbs2;rwhKBLIn)G_E@BPw&=G!nc@JffOojVy z)sk=5%_-aeMt&ec6$1!QX0d=wV{jcXzF}S|M7c+Z(CZRnYoOdO5SDzD2nJ%S!sDPB zw4T>c&~fJU$ch56!_c)E_ntJMDEcj@Rtct;gc=ZZPX=Uf`t}p98UKP(RYl1BF%vYU z&dYkfvhbwz$CLWJPIms%Q?f33vJ3B%?FDN(uSwCywlhTBRqw9@I$e`BfWda1$IyLB zU#4^vcLl&m8ZiEMRQOIbmpqj`%S!RGZ9oPWRq@^L*c#S=LvEH4Ggwc1d#8avU;6`L zXh_`$U=Ps1m+L){{l@qPk`yxlaPoB!Ug`}<6Dbq>OaLib45&E;b~M~%W~!F>A?F(z zQ6Y$RO*!yTsx+$F0lLpI&7K8e8o4q|EC-+;Fpckpg|JvRlhDM+5cgev>s2;i7Zgy#RC?GCLmL^WnzB+>mc;tkI$=aaB2_*aSe^-4T~&lCDr zcNT$wjQbnF3t6B)rX6QuL7E@fjRx?hk)-(+T{d#zvk%Mk)W;{QvC{GzJtnN{>x0ghyieUB-IKxleCyb zdt=o$D8wQiubBu}Kqo8rlG0f|4GHUXhwbp~*u-bfNbWYRizC1ISrR~1S+MNH#;O{p z$_9O<>*?QI0|5g2Bd7h$HB_V1dMla0u%^cN7EQ1`(refu>yAtemz<}iz8Pj9p}c%Q zu^K6hlGsld+Q{9WZ9AiOoh`Us6%R7>9jwl5ptbz zvjXt9>;DB3Y>6qhOvKtY$6h9BYGN+SeNlSQ4x}`z?R)EgM)Ig5N^bEHeO0Eqo5%9x zv~80tIdnH?#cEio7Q!}}4rmZZAs}Az&fB(!!oNYwN3^+bjC46l-v@sI4?Mi*ov$6! zuUW*F#aCblu}Y;%mTOkHc~tsU!%-gaPXT zcE(nTmUa|9HtSs-u$Qmx0iTz*9*)xF!j(Wc;Z`u0nHstk7svQHTY%nrU%2CD z>|HeHtmep5Ic0@A*rdr?LlqBPD>&+Y0l*ar>=W!fo%pHljzXTqr9uih)EL>#&fH-{ zA0>`qg(1in>?g!zwi@@K)&Wg8EzxF+$`FhpE4m?-RoAC}|JHR1bMwI`{aw+Imw5h) z?e=Pk-mWt^$MZF#^KDQOE{p=oa4tb8~J&GWI^p_@r_{~}3gqJuyFWufkbdffz zV2VXe(awH6?EuLsB~H6Gow072uxdug9_a}+Sf;F-mmLY_k#7N0z+&-2OrGLt>^W+G zWDEMzehEyW2n0g-usEkySrinAOH;rdEqM8?0n|L5B}+t;UKUM+5r5DCV7}IRO7{XA1@*)u#+6P zx#0&c=T@IQ{%YZP(T)W*w~ zQE2XoyyWti&&NmT7p5fn5{92v#s4JkBn$PiqKItT(zj=1Bn4YMFeB9)AiI;1_J*vT z$NkR0v!4K(vU3_Z*AM#MHO(>K*7+fTS2xdl5SX>#K11^3r{*YfS} zfz=0`zr4lhp?MH{ny}m%LT4h$pzq)L42wrNGT~8rjeF#CY&Y|>dcBfIP4nZ!9P9<< z>b0G0d2xQ|KJUhJx7m?VrgjGx_pe!wKJA8XnzP^Sb0S#}i0s=NCKor>MXb~tx)abn zN}gB~)I?3}2h*MlN&S3V5X7C7+}`l~mHhWfW(S@K9xI1?$x-p|kie_jN#`yEe{%kH zXQrx3f;CSWnP>XZ_hSoeYOD>{Cr01l=JCg-FZ-<8eN(Vw^Z+=w8jrin^&EFWny6hJLg35KtfJ}=|`)5(=UVB zFJO8QgT|UZ)y9p=PX~?Yv&SoDdm6@)!Geh`B}xu0n@cDxKY;@(Ieem@-12hWwy_9U zYC-g6Jj?_4Z90dkN%ODgCrsrgdi#o_PO4lpSD#E-ah859g#ma(PuKEFzSgMUMQ{x# zV0PSmjvZO$oRNQ%;unJaaigK745(-lreChn-7}wo(b{ zWXIW(Zxt5+W%*5_1wRTss##$JFAp;Dq|=rY4&3iWl|RFR{l7vv*Y8-zDaLiTy27q< zCRjr2knZ))r5q*aM9X&BxE=RiFvp_CT+9j^^WDs@@44rP zhqJS=a9lC!Xd7TUbpmjrd6OHR5Etf8HP{NV0_4pl-QxZ;F~ehIc)E2K`;KLiG@i4I zzs$T2M!eu$bj0ZGh(92H=^s6(&GJ$Z6+EPniV&DOaZ|END<ZM! z;Ts7dLz`NwmV6SALw*v0{sryKtMvU**h30w&XEhXd3qB%?`SRrD{9NWYCwl(4f+VPdbdGVVd+wh314?5fWBLr%f2K%%i@C;bF;Isa1r`{%y*=kNK5kxOgP0dQAV@;}*gLhMv(f5sjzfnf%Y z{&4f3$U@^t9L=(?2$c(I5_+pOJ?)VUth~hX_i}VhMwHJn1sN}Yizs~?-0uFVC+Mq$ z)yx<^j~?Qt#aJ47ed4b)kZ9@RAw=vkuwQ!{=?WgKBpBw|f3=i5D!jv*+3Xx|XG)6I z5t}4yq4S@4f<1&BCKKvEik%TwX0Yf7I8qoy z@~E81*cFjhq+ly1dKf7x!O?TY6hU1!n>7YkB);hQeEL~J-6vgOO>>^_1M|?s_?9_B zZ;yELtpq#w3yi`kVZx&uz-qe5Y55v#cG;BKx>qEd)Hx5Z6l$Q z{HyUjIKf8b+OzNW%PGLA?OLF;{ z*}V{bep_5RC0e?i{Jj+I*|!*XP+Pt0Wh45L&*A*!5Q!@pihFQ?m@ufUKbw!&;hPd$ zX@TBL(*_^S_8t;zz9TS}_o{!J( z2~if&S~7*ofs0?t_WovXxgybdH2x1d{oSPslV`U4FH>7b>#a^vydtwWDqRPX@>SLx zVJHSz5p#NYce@9T2pcC%@;l_Vk9h>=N?_&^eoV+O0HVwXY z|^^7=PZUGwl5oppRsQ*+4!kMITcIwi!v z4S5U??N(^^DQ)!j{#iLNu*#{~{k3~vt4@6(Us+hGOSsG8J0vxIix-b~P9N`{yl7PS zh=D$ef3}1i_LzNcj9&dr2Ve*_NzO3sa=BBT1v;K~f~DKrd1|c;GwN%?jzEgEJ#u;d zv*ee&Rt6R^p<^G{%EV^}Q@>>YJGlkIKX!ngQ}TKF$iiqomDe2_SkF-aCMH^Z^*{_E z6qo0WT0g49X}I=w=pq1jE-ex4I!RTWnj}Rc4zK^jCGHr4|9y!ty(Gy@{FLT=O3WwA zx|2!KW$$Bx=$|F6X{;IV?ZK!6|GuMez&_X)I|5JQN|fO8hswpksoDG#=>9||p&x4k ztD$jQjU-i)p`JuDBn?g;$Lgot!N&LRoHjjn#Ye8#bI=2hC%2TG-{NQgPCqePFaU8twP(3s-S z>BciWf?8eUp=6-2Bu9rbkJzF@18RT<{w2D??$ytMSN)Gvn8oF>@*iM78|;oNPyGh< zShBVV_49R1Y_H{HUBhpskBvn+!UP5b@P*MZ0j1e)9qD+Ks4Av+$~1RLyJc4nIVu?Z z+2+ODM{d-7wFzs7p+JtT6}QT^0-0)MsXdsT?EbSUura1&{CG1xE$V)ChToOC{T)97 z+h~k!E{V8L+lGfS@%Q~yBtq#Gw7-?6U&-FZEN+^{ldh}m!YQ&o( z0>;^j@ykj~BdMY*eHH?zB>t)L>G2C1_H+4nFC64A2CSBDhp5yx!?#3sZ`>lba_wVI zQs)!CKhsNww+iJSR$o8QN^NlBMCG&DxYf0CCEEYO<)qZ88@y~5JKh;VBs_`&6h-8p zmumwIAsx3lT6Z9&s|@n5$SzX3IbFMy!KiKMBr|w>`o32xZ{gL-39)?kudlpwqLR}# zNR3?8^ke#Qy=Ptvz-6fHH`%Io~2*!sq5M-ViN~s7Q+6hh$#HGy1uuXO?Zv10x;G$wv`4Eoi7S_nyk~ey!*X6ZfF&w{fP8GAO$qWjAg5 z+YQFRH>B=b;9oMgpqZ#_r+8RDdP)=czi@#V6Rc5PbKJ5&=C{T|By1D^uoPkK zVq%1Il*$X^A=x*IxZ``O9A(^TBWU1)kRJqibpW{V`t_~7ZvZqHYw$g8(H|y*OJgd@ z9n4zZR6av8Geaas!P3aug5lVd|1uJ~<3x>={2v(m*>*m{^_RT>C1&?M1jysHjFlzU ziX4bg;)Y7A#mH~FgSm%WY4v*P{w&e#3CMO^y>_$fu;sVLe)hd_4Dvj2@*@kAtdR)# z8PiJg=wpeVG{PwQc|6zk&ALs9(DECk0O-tCy^%J>%XtgEdK0K!xV7$7a})z}f-BuG?fD z3246SmJSf9T!`>Yo!q6LL6hCm4FyUP58@P%uEda}XiJu88|6aw1EpzQXoN4{ zkLX9G?gei$JNGa#3T|Ew^Sr0A6>@zQu?|P|gorm{MEB$Z*{>g+;dgoz1TC+&{hI{u zk{O42?W6+_{fix3kI9}QKt>#MztNlyLzo<2cL8c9*6pUP>PVw>_ROj+-*KW`#yg9* z_iv_IxTXkKYj2?sXQg{hjXwilo-9#bge-$cCB@>Ef+W2U3;a!YA3XPEI<_wYG&7sc zV%nq&@Z!kqPNqA|S@(+7){{djEsq-?eHE|Bm^8KdU23U`>iqz>{_a1a;f#HTcfjD7cRo3JoiPnr7Rn%`3ZL zVP6A9NBLt?y!#e-r!eK^l-wT}xJ>&V9fk3#Dp}o|2+nZ|k%xTGR=ZF?30l)_$K%Qc zRW#fEOz|o zce~7S(d?kKkUM`it|}S}g)(wnBKlM-t!rQ)mZmPeFz~ewc6cz}hOqWx#7z#KngJfit|byKc>KqFX4qN~w!!SthXY{iIebt=@qSo@ zE7A4?^>q?|cjkUY=kyS_GU6z*o5kd9ZJPQ`o)P`y5@j>nxiEq-pb*CnH z_RFr2KODlFBK2Q?kY0l>k|mDvq|_H^WZu3o`!#_4cEXju}j@UfHf2fvST1xUEXP3?VO zj8ugPYT$|3Z|OgkA9J}=f#AuzR>~>U5{7sQYx$KF@t&RRB=d9+>$)IV!HGh;)`CXI z<~Ew_d*G_`JVBeTeXb2^iDc3}Mv7mh2h3gQdNm)19+F#*t+1P~#%RAO{KSVG=EsLa zyOG7I`<87xmp2-K5?8^57shFf*p=5dOO)z9ROW6a&bg*zs;43G^Rh7yT>QzbWk^2Y z`p6&GW|7}))FpOh=G$t*r36+xM3Dh9kG?ha@1wXUPIMa<7=`86bl*dSxMX%#fo&+` z>yU})p9AUxk5uTNKjKQ}0%I#bsb*RGjNglW!9UE^6!>QWB(Kh>F1u2btOu;#hh?SN z#_i`?I!XB|Zzy#$vU2=Q{)KF^4*=8h_CM%LqD7Imqq`(Qd=QD ztr{%0bk@hF1!gfDfplGp>xZ&vNt*jd?`B9!iLd^pHaTNfho9I;Z!M*MjL83TT)b4f zfzZ%XHA*BN#$Cc8Gm-p6^2DzyRSj6}57j20Z9CP97)@{+kiTfX~ovj_2e zZGVYBwJw|`_+>th*ayl;4QTivOzxpg=jd%rtf?mNX7XiMHPsG}Qul20s@K!7`-h(i0_!a_gg) zYsv9Hhn$qo7P{+K4Zm54y*zubY1a;+5`E+L;FSwaT^4$3P8G>Lqk3U-X`cP4(f#l{ zL0<_oTqYGvQ!Qmng|$Cd__0#Qy6)CYVLSG+p8Y{;+|J-mauPUna~{8R!GmeQR?esS zr$%qIgb5?Z>yPb3hm)hlMvlr=(nL*Pr)(F{s5mq|9{g-oy8YxK1pCwwSYH(Q(Ungg zCilGpy%>zeK(hg!VeLql<=3F2|2X?daI-<3a>m^9Vwxo!k@+>20_Hu+ z@NS{$Av_-snpBwo12PfN|vIn1LF*3MHN_xUZ>4}mAleBk=6n=R$? z{Z5=KqTST^X3o$Vd7dGNr{irUL0fgIG?t#(_rc2+-$Do&mOtj?bc0I3Y?(pn*4g*J z2K&tY!_Kq~9-lmf47LyoW;+jIb5yOxy+$tLjw>3$IqdZR1yv6AXmYUVs3c{fIQ8ctORysEd#bfyAqc0o?U6>C$?!+nhcfByL`zdO9w)a8 zAQC;>v+_zEuWwH2~Fqq%Da(O(;%0Ih>Mck%lyWy zW#VQyIph4asj6>dtjQyn!0OJ#L2O_CPX1u3g!*F$;dnG+k!OQ~L9w@Tn%?LkzYsU4 z3)OSv8%X-IEa#2?_wH_g_p=0LSt%laqjBDr(PrQTU~pbZ%!}gs%XY1^h{|gu;@YTT z>nsq(M`uJyN%SmYJW;;K`^r7BZUb5&#CnI1hBF|?g$jF&s~KXGo(}T4l=9<~L_6lV z6UvU=ve3xtz}3(gv2}uPGwL{CyV%Z*r|mES6B4nYHyeE{VSOJ zp_}EH?9~<~T>sW-BpTiuG{<3&;vciVeVqhM zp2()f?MiQBUs%)NEbWO82ZjD874CUN)QRud%<^VWzeT96{w1O@%3N?3i#5fFH4gXs zgv@H4#FkWeoRziAl`4?=>7v`6`Zv2FY%VRThN1N#;uM5%Y{b491tX za|57)(vGczvtyzaJ#?>XMS?zz^%D);t~CR9+#m>E3S)nHn4LtNJ-=A;m)Zo^ORRca zzA@TQwd}M)3`1<;s%dj<)Md4yn|DizMnTv=_Wm1x6mb;y72jVHm|#Kr#t{tecsFn< zy5GjUBH8G{+E~^Pe=YKUAR^{_!?UxmSTIWUZgv1KAMBX49{F`M!f-xhq+y0(#Ckc* zTPHaS_Rab@MRzPhcKUWL`=m*obmKy8A@*=JRC5%ZayWPMd0y?%uhoXD2%{0yypsq4 z2nbqdp&d40;Ph$okk|&Qe;(_i%b&zPBHHC(_6FC~3Oru0h>66U{A4ula>FQKh@<&; zcZ|>i%V?O zFz($}7x}#Sa*JowT;@k`DO%mo9pqif=Gv9V!5@q9#KYw7OQwm0KKD)?MRX#pQteBP z_3wRNC-z&oZ31?h|FKMo?DlBvON3CU9;S1TW)5Vhw5nYj2E#yJ^l;&s0f{auDm3g;PX(V;cq)KF)1J zbNq+Qbb@k26go`4z_H}@9f?|35f5v(-^HO&6Rzgm0serl%~0S4*Ncs^6i?#x>xI~< zOY)1&Iqv+r7)TuT@r=^7qzBs$X7^h{8c`V0%8&R0$A+kRZQ@98@u60|JDE+%eazYy zwU7lpI;7XO(;uG)-aT%{cX$r~DZ?0KDlI3UF`MedzW#*#P`|!Aiy@n{GTSU>t#DNze%&Sv95I0(v9NBwwv zvZCB;0PIx=M@0b82$J4qQM7l(KGJZ1Vx-xy9xkpzR}3S@whbCO1z!zyFWH0xSwdSi zA+eva{dx6|H-9ubjVD?qeUvK6ZTpR6{j7amgexaqYT_~JtHt)=vPhLYol&@miLA2^ zhY;)?k|^Z%lobNE;!660daLqwLM-Zh%#X$V*4Y_)IG_sXpU-vSmN$%5M;x^)ukOdV!wHD4CH zY-HzxfuLX~z4^%fAseogC>nNbKYxY92;p~_QjOg2e?7YcDsT>tH*e0bxPPvP;*gQk z+XpY`y>|r~>6X5C9lyAKgD4ls{qDsf%CBp{7ptjm{~Nm(OvIsgSG)6?=W@&=?OY1_ z{tH;H6_nPPkL$wP5o&_0DD6wcG`gwqrc}rwgequqr^agUo3%(RX(6Z2_P@&O-*A<* zZ(~$BO}UMF`8&_!%zy1Qa~%zxE15~(xgj6Svm;_~tLVWL5r9!YpuJ(Od7`wseUA!M}K;lcLaCxyH+i!P^Myc#A*AD=a!zR$&y@i*}OA z6QEshc~=9U#Y^m($5gP7d=6WC$ca?W=Dkrv7slk?6A_rIx6BK&ztXpQ98Menw(yxJwj)+}%Sdy9`=Z)E}s_~$CX zfOO)&FGzNb9lQ-yOLPUXMT_MVD$y%^_n1WHldfNwHY$e={=`^{&LU$?D2?N17-4B# zaW692SJQS$yEq3CcB}Dklv*}Dj$0&k02BNYCH|=6e8aTGVy@2CZ}^{w(3ihooB@Hm zuEw=_i94@W=+buvk|Z?Ub}cZbB>v)pMhlTKWfZ<&p;k3uX!sgOlp zfC2E`mKq1}wCY31Ivhjq@I8UTWHX(OkWUzO+V7JQj2)1u%=38Pqq03dV_%P<7F|&{ zz@wP2;Cwu@lt;<1_;{iMxF=K-*^_;jT-X8tsb=Q>(4~NMBSL&{O()(mrc{WnW-tcyhXY*(3d0QiRj+&H9ye*sD61GCG!|s)x5`VE}GpTJpEj{0yLnl z+^@W7P&)}+>B1ig4suE)I}QB6Qbsljr|zs1S1L>d>Wd$C0)uYz3{z0*+T8%Rx-hu~ z!#GBqYXeMJ*Oy}XUygLj2@jVMQjY{W8$o(_GjKu#*psXNAlM3#5|{tSH(n+@II8xh zQ%I7Cc)HMEkN{^)MDEF`k;mE>+O-!o=U?V4Xgt87X0an!uxzPTU4x8_dR$i+m_N8ux;D%#-N~<66whv$7#pe6R zP+C_nR`o!4_gLAZa)Ip$`3A5189<-ErD`4sG~pG+1m_${XJ89WGUH`J^iXTmhR z%JF9n3iifq*)~0M&p+b_!To%qkJx*GSLj6ExxhMaJquq9HoT%K{FOXmuesn&eHaSV zw!L^6xpoE$qPGUYGD|pjy76CcEg!;(($jLQt68`}VpRQ2Jwn}nJnjnbj5bJo*!j8Y zd(W3H)EsRldpS?9N0qVS_pR7J!AUs4cI=h0CtQCRuhE%?Ygb%ml$G_$x zF5aMROZN)xfyhR)^h8oK>e;6#{12p-KjC%9=@eAGEU8|*Oz}q#zMstB<= zrsPRH)#nRh%Gw~0sJ{?X|FP&NICtB&=fR_bRl2wONs3b&w*BobEv;4z|}`?+Z^pMIst!78{S=UA~D=7EV<)Bwj5 zf+QN1mGWJnc?H32-VzBp&6!$v#BC-p;4ICzuA9ifK)5>}=r$pOJlTt2gRKoiCe$N& z@5W(x)AkhTNO_$ffds9qAax0MsEdZ;r%dQ6_coy($HrI$Tl*3l?>IZV;>a;!aL<*r zheJ6~TzJp}ATsBzT~EF8Z1_4tn3V|2euzrrV_ zL8MC=woda5O5k-_S@{Pzg1mW0zXeft&wf*i`grda!a`ATM57LLJ0F)>fPd{1A7a?P|+tK3vZ*ZN@Uq0Pr^ zJFSmtfo0ILerA>8FRHa5EwK}x53+1ap!(N{_#9#!vdOF(M49f^ zbTEGK)sF{{Em_s%+qTaLk)n`55~ZcUURnqePYwB4I3l zX{R9hEnrXwW$5MIV4y-uw(gUwm#OIgSY#s%Zey{|W5IBBZ^Jjpe3M~D9uXIB0*MV5 zo86pL)u*%rMT-F5Qcc>2fov9)x}kdEYD}zwDwYA0OiYF>xQX4S1ousm8X!Ky0HTq3+UCjyAIg2TtZV7AMxD97aBDe&#jW~k(w zieF`~u*wz;F=%n`xUCRM>zeUGz=SBMHl}$ zJx+eA#)tUMD)zib{0R04_yO(ZGzJ`*}IGK=B=+JZI|}3h*K0Do0 zeil-m;O@@pz0UW7Ofd9ro&SW6@5VIdJ5Rf<#g}F=E<}mX2`*Uyc@r0=r{=v^XBijn zyg$5-23`mp&t7R=nJXd*MjogEmR2UHy0jH%Y>4^69sB zfiY>=H>s8Fa%MU@Fz&F^{ZQL)N|9lw@MaMU#vwP8d``Q2YvbL|fk=N9MgnU&jP=9d z6QKrREi6+rjbw88dA{V<|8auL=K@9&R#{(=WQ_t6yl(Ga)-DYV^0CgrJXpkbZrR^im4TVzgKO=<2sy%6?91KKbO~wPb4{dOHM7sRGyAY|&f`5$I&V zltv66-IhZa@nARO65k-R;-Znk4y~Pra$b@+yS)8UdM_=Tl$Mvwm$Y+j8@0BzFUi8q zH#<`NS#IZdCCC*&3(BZcX?h;-#HvR>y0o0+zsIsb`(9AsEKdrjDQQzQB&3U zDn-0P4|UcPiuo+uG<0o@7xj!;jCuE(E=6ft8+AGY({uZ4C#VeEAJHYN-MLmqB7G_1 z25;x8!wkOp+b5o9UBSi@PGJ5^1Tj_XC;+ZtfSpt$IVn;4sIjWv$`15#2vz6*mJxK zQ1itBSuK24>Yif~R)U^`;2T>Ul(*Ca0G{9tA+eDt7st(XAqlW{CYC0#kg%w`E}enV zfRHz9cJxk>k!_ipf6#naMmpg@DqFq?v(Hapaf(4syJ$goQ ze7!CTftM$@k`wO8Oy4h|j3P;qc*la9iA5-3p=4VBraBy&f^u_Z#188sIN{GRFA>V;7O<`tr`Wa0hju~Ti zR~PVuF71yeaC9+ZNh>z|0Br-`2A?-q%)n!Oa-EB-jncuZ66ICY(wEd3ZSuud07bHz zQ?hWhUr<<8k!drQy=DjO7po__Dr;Ji-0IQ^qL3jfIglhuPXszk5yi zma`8$U+AZ|14R`#0sA)))yDnDUj4rdApcMQE%*kyleb+41@P~K<=p@BBW^$l822rj zyi1J#UK{n_WK(bNLmAdPUoi6T)OG*k&(m)qb@=NjtpKPx{sYBjFC#qa; zfG@XnEn;L|Koj_BokY+aRaAdu;|Mu)7a3$xSk?|XiB>$62+CptaU2CT8*Jbb6s`ON zly+AU72D<^j1YtpoI64I&_3v6aEz%mK&@a*$y2j9$16KG~|fOU~_$+r8p zG0@MTGyVaioomw|8AZA@T&6^eS0=CZthh^Gd(G;aN5oIDPQmcaIze1(!(D{R8@O~Y zpg$SRH;31>Xadqp?v-|^K{M=UuPh_6elNg*rLB7nNhwT!*x5VgVP-EZ@7M6fOk=| z2fB0g5^|mfy!=3iTpHr+gos^ncN9PLp10WQ;sShfiR z($ff6J%NcXS1YjwvCf!yBm(1&!72hV4-zuByEbtz0a}j>d~IsdWPQ2YWo8%}tBoDqqZmYfe{W5W;$JGX97}kqO2B#VZ#_drHnv>lD=fc$pxiycg*Yke~j$7!Yz~K=uS3IB=%e zjPBw@#YIppM+%!Xp`|zx6ZB^2_iCC?t&BrR+)d}`h-I&qIKlwYwEUOlk?a|wsI}tL z(ilZT{yr12QroUuo`%F-b|OF0*(KF$4IX&GlP72d4xuaxwCcM-F{Bw@2lOgGH$m;- z@bX3>jCHuaiQ{EOByR$;zv9@SM@sLo#idRbh-9LW=@q}%_J-FPtR^otmfx(xvTnXLcXk=eN{lPmRC{PrR0?>4UsB@Q>W=zl!-3pChM8DYTd5=5bhP{Ly%+HHbeQzw-= zcd=8t^9$FuD4q(8b2b4>4=pQpw>nX~o37jTH;u-3G%AMMh97dhl1eXJzJ|lFkzHwD z&&3&zTIxPU7xSWUbZysHUcy$s<)1sM_^Dw2ELV|FTci!7=fWt|KogoDavVOHjg!c< zqG^iPE2#X_*JR(F|EHRAW%XxqeUYduO&viOo}W+bwUVHK74`)TZn@YErbfq4!g{rF zvbY}Rh;h(n;pdpn1(B)vM2~9fPYAt4@<0Z=l_r*u zI1uFFPt^9+4?wtc&AbXELGNFxd5(@9-4T4qlZvI4;n@;!jx(DjUw85Wc`JjWOdiK4 zBEI8}CLX~wRv&=H6iZaC{*oXbr8ot9J@WGKg=ZO4jTIBT_r7wQP(T+4F|#oxF6^%% zPJM$8MWUBJD-D~sR0jAJ|7aN$X3r@88H2){5DbMxKITgjP+aY+8^Y>d($ z7EKf66UG>In`h#`m_Y*NWa%=c=*6WoG_IaSQ=G_TC~HiBj25krqRqTB(R5cu?eiTF zXqT~t=h0MXy^Ab*qVUoh_&Wga5I>Y$JP(&QHwnfQ%1*xE|u`%rUOE&3-bo4`- zESR@k%flJdE*h8mh&1GB3zu#^U@1e~#HbD6WsejGwvYp~O!k)5LYP!FQ?}%Eg}ks6 zu2fHa@d0$(oo(v?T)s^fg5biJrBN9{I@xMWIQ&bOGb%f8k~jz6>$%dUzehIy{PXq+ z7=m`M3Thv)FxeU7upu5!%gp$1704P?2C6<9Uq?{Z98}Rie+6GRLhJgz>Li*!7y4X+ zWu;tmm<&6{+ER=Mn>=YT28oVM;62g*wXf!_3p7E!Dmvy!t?&QfrpdjoF1wauvf2FqhrPFss&enzMz@Fx2+}1=i@KqO zO{ut3Gt?cm64yYURPZ8w;IL46UEQ4!UL4m= zUnqxq)<%)z&VIT-$@U3W*MShJ1!HdZl!TZe}OwDSZ0M!D%Sgi%~+?U(hEPHt4hvqg3|yIT5};ndTT` z8+i*78yJGrhZ2=`#GBknl}DJi-WlGhxAbGNY8;jX&9>zgyAKiTuiP2v3hLkMIMYZF zPa4E#B0uK^(R-(&eG9g>*@D>cCtT7j)S($3_g7$bq-7Mh|LscC9vVWS8$!-)hF%fS z?cjL%3@8)bUV3r!gtwZ%WJ)ZYKiZ+l{z6V@X@%j>^pQJgfH(92eL9wPVlR*y99Bws1qM` z#@F^6HOj_z*csN)Em8w^`HE=xuRTQGtX;guW&DpL^a}w<)qyY)Ob`92>@6j#iFFJR z5fhYgpC-*b0|sN`a?G-{ATtWvv8KYgGU`sH{^>E)XQa>Uz&xOcE&Vf-8K_1;eoH&&B<$E!h(Z{ zioTku|3v}VFI;PI8`v;bUPwVAmfbH@0JwBYR(RilG^8WNzLsUtR(e#&Af%&~CJHA+ zEDuy|Ukp{T6oqE}KUPma>=yLvsNzW30su069Sj<|B*f{-ZuQG#(`aj)sK41kh$yGJ z@dks{F0gfJGhTWzY0ozL?D5=p6X2id<`CIQx8eo%i^Fo5^{l7TE@4*Ep3|NRTeu@w zorTvBvL>5c{(G7+e39Ei&B+(ZPubUP)?aafp+?(MJ9v>qI-{re!GX2?S3XV2(>K>y zQ5MU-(3rVX<`Je(vnU-dlOu$n&HYJ9c6HY|JNq# zvqboz1chE?j#RcE9(3lO!^r$5Vr;WxDKPNrgO*miQ|{%R4Uij!ADts{_1xIa1$)4M z;~hz5uh6r3-hzW98|9uuGAS#Pa5lfIZSWzat*8~gIoGGqQu#x! zcA#z%60FOJB?L1tzR*N&DlOZQ$pGRRI6KYRsH3Q$YjeFaz{o;f_&gd;⪼RFGNI` z5tIe>InVl+`o|f#JMc5i`78vEWt~Icff^!y9a)0(Z{yo?_VVNBY68+;m5gVp)&=>t zF?<A9wOT@$7NuSBfM|`T(%0|9;Iwk&14Ln3-v` z0YZaZyZwcA3&lVIylj&st@cQE%3TtNT~nm-6R)R((3%C#A2o@+Jr4a2E_j|vvyOcV z5Dz0UWnfypRr9^;=^zrPiF{kJh+BVO-najUm)DA9L8gMW3^?bceqkn4&O!{m&9hql z{U!2BomyV4ffXj6<40+10r-VU!B=sFzk;8`6P~Ejt3$9u2d7?aly54?)|A zgQ~*0&^qTyOCU*z5{bXN&U4`O&4Im>X>5807u*g3(8Q-&F>Hz2Ka`o|7eVe-)iB3Q zms>QBzYxe^NYQvLro?S)hAxDNKl8{`9=Zz;!~ulz65|THQD3)d1kd_C7rlRJ7@?Uj zmo}$m@FuoUsoru-Qg|_on)2?`5Td_*Pxnqza;EzYE@R1{TX)(C_Nh#;Oi$h`XGiz_ zsDj!j4HID^Yh9i*(qeLKOo%h++aH?&dfB|E-C62RYiiH*I|~MD$R@)<;-juFd|0F? zxWOuD#C!og3|iShvG7T|`7TyliiqFm8Ew z_@VSN!<&08okdw+a!DNGxQsWfwX-D+8}k2Ns=gW?RRW{Kjg#KH#izM4)E@6Zhqf?@ zMsdIJ&YGoajGdtF}MO9}VrXm!7M*c=2Er-^Pdmf^WONnZ}xo0I9sv_BOcz*f3Op@ zvd9=CPeg=z<$o!mVPFM{0K4Ix+ac0IueQZ{<=DkoBBCS0K?@}+a22zs7)l9ld*DV< zXWU5iYPxX6MTvyy8WH}Zi3j5$yD=_6?M#ufo0$Ivmtn;E)`pY^^+llAY0$syC}+D8 zNz;g3Vp?T8OeN5#CNu+hf;msx#ip2i!ZwC(OjC+E_h%rOGr{$pGN%(lnlA-!(_F>( z>877RY7B+UOB+5BFt`0S!+sR-b)dTh0FiRqsy}wPoKqhRD)B_X4CXoU%oZ|!zO36B z4C>>%Rl5c454Civ94LyT5<8mxn29AjX)8nn&ETx4>;fU#aWXP%GQk{+tlawE%~qmU zz>Q00tG#?ZEe`5!KLYK#m2@vpzHuCrdSH_EUMJP}uP{GDBf*Olz8daTlotYzG{|;k zKeNq#PLI`%_fX94Qop9;mMkOKerjK5X zg*!nYy8@T&b5ssmhF=%IXh=jJ_}_ zLNF>Co6^+mmdP<(J2BdS-vs6bel=!kSu>(28uUi%UP;mqP#s`R?|rtt?`{09tzDRI zak|{Be4@lUjG1`)Ib94^&%k&Bk`Q3+hC~yja8p#qajD1_$&*XHeNEg%(beeJna$lL zU7smbmRtVyd$V6(ZKTqHNPc-o=R`#0XzJ_{q%E*5J|wKKjM9dU`s{V&YY`JAYIvq^ zFa=Mie~$V6`Sfb1CNA5%40WJ@_Lwe9^_qUzvfayvmbnmBso3WE#ja`N>Hvy$cQ_bP zlyYH*NJ=MhE^nV;m{iLBn&xhLHbSuFS74PX?*{CfZZSYA7$h)+^^WZ%okueRg{cnHN9l1SK=C;H_nUdrF`TjrS32|%6 zLe$U3i~^<7cQ7~S?ud;^RXd7A3(=IPzXA#k*MuWVV4UY_J-e! znLHdk`&_sUyy3gHAj@PLK#4Q`rMzK=SG+j`!Q~v#$gSHYc*A%vhE_QJH9n5|$El+m zb5S|TzrNfoiZCsT6V=od`(>vQc>|?X==@`eIV^_Q3W~`S)<-@ax!&|&Vz6BxvZGDV zO42NXyt{i#-=fcEi8%uuOeIUfKXwulwlnSA?t`Z|<;uDFFE%^Y!onx_yS1LeT5+~$ zYhkVZEt!xGdhtyksFzUgjfhx8NWh5U_FbS-*gMXEU-Y6`_!gnum$cvC$JLk!aeCud zhzkeNi_2-?%~w`7toI25zHHX+h7=B5Zg?X3H3Lzypuf_AER|h_a&PJf zfqxa^THqtF;1(R9pR~yFeT=q%9x#BE=*WWN)Q_`PZU9zZ6Ub&}Q|D`6yn*W%RKepA zb4LHUVe=1M98vW}vwvIoDLw`5rOdk0fUwmt|5B2zMh8=>lW%b*pE+OEP#PIUePm|1 zf_I&S$Q%h?e9MLLuK{P zXBiiuy>@sAFm&F&MH=Bj9jAR@Bp|27Ns{fcB1ELq@;IG>cqNrIXE8THky$M^Ii7Nk zDda1lZ`3Om3S1V=k$liwh#6*EzjE;6KtyGk`O<=!kcD9Alp?M*2$-~M>_NC(3v0n zX0y!ThH++dxe#Fb^Vm2Rq{}%M@r5}1A8Dx|3LCFqKQ~nw`3B%Zj`0-s7!1TWrQe4N zg3-b4XcI1U?_aX^mUKeWp+qA`&%{08P9-rL^&wn!#6$r9qnmS25H(djC1YH?yNG7e zb#);p6nl4ImfFYp9gsE`8$Ab2JDIEOFBssfU{zBIHx827^8LhkvxiQJyXRI18ooIQ zzr7&>T4cUqr(!mX`|~~~D@=zm<(^;rF#xf%|Au$MIih8<6j)bv5y?rju1|#W$iVAS z1}xrNSRYX3TYq4Tub*W~K{NcoBLj;whgGqw9!j62#*^X)cBDnT(xVil9tbitX1}Hi z%YzZAYmga@)*E_!#d~v@TT$=@5Q}>^ZGMK#)FSp-KuD%^Yi6nYreO~sS%j%cw5y zPqwpI>J&Txn)UZ8&%>FiU*pr*_v=szIGGyJ02Xx){+N7IN|K1sF!r|Cst222x96St zD9fIH;F>Q{DYE$HnLyeuLa7+-qb#m$*K&iD9VEC3!qKA9+7bfQ-qUP!W;vq zh9d7Ufv+}$>V&J~m>VAX^)JjC$aYYbP9~rN%_m``!?g1V+-Gb9<+e#7w_>o3 z%O|^tu4bWmVE~xoEpMw2ofz7dZ}dr<$vA-sAE}fm7V$HX9r$bM7`8=xJO^BcDvO}4 zYy+h3{3E;1*v-Jf;-3>n&7FHdEpMSOhSu7+=7ST$Yov|b_hf+plwI#m_D6l@Mj$xo zZ2t;Itdu97WEw^8l81HFieQInT27D;(PYY)h&_w@wYK}}Of?o}BvNVHMQ$*pM4jzm zmj~zp%`9#HoaaR{*x8HM(@z69_hf;8V)-{}miNayP(`OhuBCS<3j~0@lrRLlkdWEz zvK8K1XDp%P0(pBapDfc@*&>v5m1w6hjrJ4s`ffH@crQ;mTyuWYU#m@n@Y2=CgX zLo&HkBDNr_kw>~H^+({S2}AI~y}+XzPk7sE_U-f>Q~13G%AztuC=#Do!qnG&KK%s>qsi`KP&fX z?(6~4EujRVrpKJkKhm?&7V0N~i*6k`Tl8Osun8)^HU6?yJI)1my9kHOvBWij&sCOB zj!zEO$*bHBhBXrkvWMHNn=ey+Ht~Hz=@+VUQPZ5z(ix*@-(ELDV#W%!;xlxoYD$Fx zpBi+vvsp~t_<@>|lN2c1M7Nr(FvN}H2<8p}=mG4uHKux+%|ARx-f(qIPo8jK2*>|D z9$x<6>Nk;rvgNm{$czj+AX7yYNLz(&vM>1X_vgO@&2mfaL^w-q1R!ws3V)_(RD^1I z_n%N8f){Zf`n7e~!8a&K0+O`G4Lgft6UrJ9G({;^ZTv|xUBzLjr zA8lFY8ySm%tdc+8BE4MtKZ9f&ryY3%$6qT+H9aa1Msz28TwmzmZ^KtCbGbKfVtIq2=_Q)}C|DElh|CMh%G=#}{_^hb7^Dz_sqLxSV3Xo>k zbSqp$a22L2wRGcU0V$gObMEF2QoaD#gnFHNBe%I;J9o6|Z2kc^4zj{tbe8$9&6pV{<(DeF0N!6`!66? z{J#TYDME_kyz*eiMwcGpev3Oe5!|L))it!++$jO$dHpy+z^yOKN^Qp-M5ApnTtYYZ zV&7D-v}NBynD+XZ_dd*U)OY)bI7*OSyUzQ5_rYkPI}(y4F)P}lqPtFGq4eKg0A$|t z*h}$dgDbgU^E0wI3ys>;Wd1A{tx@?*r7C#ef5UNA#`I*quzc%KczzFtz&rHK(&fVC zVNSBycT41mk`k+wHIq6?Dnk%BkbA_wkpGoA)@9yhM6G z4-AA?_M4heWq|)y>&04&SH~lBDR+MuL3q}{5gFfc!F2}e$=)>djg}_dyO~*kOD8ki zahPV8y-!4>O^jG-2llsU1E!1EFjukSvG9Vi@72%` z#ipN1D7ymp@;onxV`Xq ztP$}~iqxSG!f~!=JqG&6Ja_EDxDac>Sa&PpP|a+!b{40h)3wi{fDPV2KMKt>nHI}! zEJrA%Z|66`)V)^V0xEm+v||O<=PDn4OWK5!WLy4MlZZVAJ6-Rx9d#yz2Cq_gHeWaJ zdsvtum|HNrvUf;LpYSLKCMt&+caC1%hXTvk;{!l}p5zGLx8THl0J=7^HYR17jhavo zGfr!H+iVBOZwq`vE_L4SPf)Gy z4YWHqPJ{A3DLwWcIl=M^gwUPDKi`&?pqHe{Brc_&2JHGx$-nJJSOK&f$arD>nEYSu zMvO&Q=AnF~H>$AbS`7d^nT;DQXedSl6lOtD?fAS~c1J5no>KCy8P;#05tWkO>i!HL zDF1_lmW~c(bIYB(rOFL3g7SB7p8|x}L)&@lW7d3>v-{3x0GLahPk+w1NccDw$Mmm0 z{jpjHRuf{057>QbPjCDp*L=RYxb{hrk%F*Obp&OLno!hryy^uw`{&VG>N1{dP7sL)(2SdGn(*7#* z#T`yW@z)Av??}V&+$qGFP<)z1_k|EqFQarup+wiKCNn9%=R-166E@0EJ#u3_Djv}y zwVz3m-Gl{VH9l{HN@aIC0#Zh3$$BURn*Je%>5k-8vt?E2M+!O;4;^5k^0ErABm+rt z7mZ!sZE6A#EdD1LQslnr#*+YNp!L`Ss)3tzEc|d#!ElMLvqqa{?eqz*PAl|5uR=!RE-idaO>VGhy4SI4*C5uhl=jU*axE$KyjJ z%>>*wKdXz1gPz()6*)pMNXF6{Z@M3lDWRMU?LzPU&ZG&CIlSOyL*crpE2&*+W^r%Y z8q%(S_jF}j#2cN6U#0lly%Qh7w+h#L(1@hJ1(43= z*bW47;5h3^KhNPy6^cma(-H=eoKrE!Wh%H3!xNfs?A>vD{(8qDIa5a1M2(VgH5eI_ z8+icXWu(3YVOw#`BVyF}aCauM94h;2y5dik{ejU00W_I;!RV-8ZcwDXMhwoY!K{$N zq>R19W$oz_;rce37YKck2Rz5+Ydx%mAi;~lQv0aEdOxE;1qB2q^o!d+sp-75J zis^2B(!(&IkD1dqi#vfdn9~uYuPvU$Jszk{P)P3(6z(JD?cP9TYmkm88+dFJ*pU^0Fgg0S7U;yV5X`_om01oU_vr;< z|7F|@UOd!psf(HluaMp7B5Em^iBxOV+*S51mC65!^wGfr$f!m10Aa?lVT z8Q=RPAd^qMm4~i{)M`)G3#K8{Y8<pnA>PpQzR-%im*lLD9SBbmI+%F zpx<7Z*6Rrj+?}B^#`=~Thc(FbEV;4zjP)el?Ck-JhB{n{1o_rGiTZ;VAO16;4Z2+C z(Uqi&BY1?XbeIJ?Y{NVm2rCGAwe-d8=6G66-I^L{wKG154BrQmj}`9%6VrWOnb)KA zR}yY-Re#P;Q)DwYsWMW_n-@2rGYf4H6{x1d|A+mKQe*A0 zviJ?K99eF}%YTs34`t^zJa1fVavSp9SQDN|o2f^8P3+!EphvcQ{E|~0=89<^1KY-r zAGu!{Z@Ye#w&lsvEiyOT$eJztL4oV+jgiXrr=0|gQh(0RY}Ve2w1guzjdXOhkyNop zz8XwO3Y0Xpp+_yi=oP;5O|-vQ@%vqFt9Se^$APQys~02xU*T; zLveP7I;^Ts-&7u%d8*Xf!HZwBcw>(D5$jew>Jy`X;{i$AHiLuez-Boz2P7eG`Vi}J z#rX})(-=NB{S{V`ytEA@&+%ML(7jD}X;Uv~02z%nJUQ_qHTm4+S2GIOX@-EPN%2SU z%KL2cK)cJ45jkc0=#<*(HUoc2oEvuP$Btbwwv&545^hC!M=JuR7>}6-O&T$8#E816o0eb$h$N1d9a3EwsE2d)S9oaW2!qsNX0xh;- zI3L7OyOxC%3}sPg?1x>7!Ut;3_J;+9#X9BJq<Z-R}Ji2_XcG; zNK+k;eb0Jv107yfWazT=m8L@N`88Z^mfGfJ|NsRr;ehur^N0pX}h^nFoxP z`d&r-as}ct>pQuW`qBE5PV9-ro6;Ht*P}!=i=$rGGvEA*>fO7(sE3+psm2iSmoKy1 z>2g?P-VJWM-Q9}z&m#!c+9h7TBDI9jn{Y+YRZKB_Vcf{TJX!Ij0zGbq(r>~yNdfd! zf~mpH^AT=|9+$Ur9E_=3eqR)b&G{lfB6Xy){J2wjk<`gg>c&jGZ7#%4FsA<^XesRV zeiy>^^+jw1g#o9ics$XqSB->k!qz(-mJ8fTj@-#_C4s~)NKE9y4Bs+6T9p%R>LXRn zF=E`wGveDz*KND~jp38+U4gC1;y;mQlHaX(VfhM?M`@yI#^2pe{tUdFyTrj)TGdBJ z_i3I}3==;1YvfvknS5rSow(3oj*r!fF?(=xj~KTnvE;)WlTAVEZ_Fl%o-lF!9Jt}n z#=bk%oJcZ0*hyn&aSPW@`Mpy-hfJd3egyG_Ih}DD$=p(hKu6mo4`U54Fm(dtjTDf# z71h4@@Iy+JS6Js`7tD@ER~qO?tJp7g#^u_Wi_(UVZUOZaPtH z(T74ZUFbnYSPXX1q64fkC zAQgrkuVow@I9Y5R7PX>6z;YuT9o`~-1<7cx*OywpM#0&HenLh4k`)L}4{s?A*QHHp zHA{^cM$z95BJp@Yl1Ud7R`{wDOE`!gg&9Tn`k4KFma4rBP8+RY-bk-@ly+361A*P| zD^Y_w9{|FotT8wcG04MJ3l9>zjntEB&zPZ_)VOY<&D1?BjNDbi>8j>;$Ad!HlU{Yo z(FU<=_F0%O$5P2FUBb>KN-_r=!KS+E9 zQ~6#Zt>DG?#plu%7?%b;OxpdJaQ>)k-ed((<=={CzLX>CpuUV3Bv%E3t_7j!jw>OM zp;lTVF$hW23A#3T4RI{3T~ng_4-`AT=?b90u|NMUKm*;n0WKIt@MH_2E$$8ajx(8` z0hfj>6<87nYMI>s{-e=i2&Y0RH~pLDpG@T|_|!MPAG?Da_Z5HrJ2Cf05hJ^1_+&Qy zK^sGPib?Z;JN_p>6S2s!+r~#|pVtsQYwT;pXbd|^hZha>=4p3+CzR>dOoJbgk@73P zmc7SACMB>`{^x&I1rUWojKvv9SuR7*=e7+K{3{5z1?g4|OWVkMpB4ZB@CJLZTNpsqfA@vpFIdLHqY5Xo-PuLJ zHBReq?+t;mg2L(KYNyC<^FsgpPu8o845Hxb$jcRhn&IDnE3weO?&8!zwV=HHw~Ln+ zV>$|1Av^i`1OLh2rvGA$yr$ACSwH{%M}dzk=7uO7-Mp>W%KwAU4&g5p@J4hueBa;y z&pypfWbfU~NIqfzkG||3Y1mw3UbJg({-aOx|3CWw_ooVuhyPzZ`U)Oj#9JZ}ch%$@KPA2ZZc7T{LEJF0_z7Q%360-AKP>tw{_B zb~^_)bHh5GEA!7Kp6o z@`h$aKuDE^rLd)dL8-+79d zyeH4@;QjS;4{Jh*oJjJ{g^+xpXkY>>@v-W}3EW4okU3U0&{O-&4`JFg__RU3UdeC< z=)vZM)ew(&3E-M4?YD@%Vl(3Av+?QAIieIBdaA2Ust56j7s^5}aZ&9f1B-3w`7Q&G zQw1?FNowP|ObRd8>`CYlyxAc|Q9KAOacjlRu>`t-A7Zrf-Q#VL|0FoIy@DZ>Bu*## zb@96J8!>e!F8oDA1If0e_1zfrD>!%>7UbvRbTo$7|Ml9i~w_C2J&=ZeUr+@*TA2Rc*h8d3^aF_4k z`ysap@5ArVmvQgaAnfjHxYYV?lr*zGB6k?k%jH=#9^R|{x~(RL=}F)VQ|3+zCLmfr zX@*cZo5f*7`ZkgSY?5>?PUJs_`+AngKG$~VdV>r&dvDl)#e?+t!xS(!lYe<(7#Ou? zxYqoDw4qt+i}_izl{>%Q{t8S!<1pDjC!?Q|&%;p@mdys|yW}O$^&n2!{O3xXc9P`V zD+Gx4&kvkohS0u2U(28^yA0O$*-;YpRrh+~dcQ7B{Q*_T?HX@uyPT}Km(Th~%B8$E z8}AC3Wng2#tiXYyL1bqZX5K?YXV%iU>CGNd*x{hES%>K`6mAiCeDsfUdxV6JzmC#W zptz$n@Afwo;e8RJjjD%V*JAmA3~9O@8xYQl#DU5Y;%W)o`uEdU*i2Y=K^_z7$T$S^ zRDwfLGEOPZ*zCLU65=sPi3Ls70;x~6Ff_Jk4oWkVL`2Fu2ZNaCi zqI@$lMDmhaI>&}OE11O}QApMNh|B|}U7In5R5It82b>o7)lT7Ey+&_F7D7g-`Vl~7 zZk;i%nMvG&IHlGjE!xf!v+Gf#8b?qjnYn6@g0(MUP}oO_Mcl0~u;WR0i*KpE)8g@B zic+?upiI5(z`yAhS5_LrSF)_C;I(h3s{94TFep|l>-blhmq+igq9;Hsb%o~`WY=bgO3C3xpp{xewwU=Jw$<|k*`2Yo45P^~ zS{y~odcO|^9gl)$H>0z*Ljlm~X|;_(v=BFqT;76+>E9Pg=g}l#&fJZ>L zV&*kCLNaEN5tKyzN+NsIV+bIvo&l>vd*N>xaY)p8wJ@2n^6D5Gol~;VCY#nMCqULU z(sR3Hppv1FnjvOob*kzq%$}+gNS+&i?4a0LRU`6E7l)ft3tzOZ`HiOAggH^SKQnv5 z4(narKhSxY;Yz+w2yl&$^Kyxo1$UBR8nkbR!@0n`Wyx!u8HG8pb;Wyj&AVlF>j zg4zVI9W2=-1XXv&HfnH9^5r-*giErw5EN}3Q>mLSFY80*c+$Dsl~$Hmvqe$hEM478 z=psn7Gskt#r@EV$y7mj6OTc$+^R$L19cpmuyr{l$nf3LSGSGRJD1hl3TwUB34Hv zHfY&U!e#nP7MWwP^DEW!vAG7=+PWZBLV5aguVDs6A)^?cdkGtnZ!f=)gf;rTRXIVB zz|y5`Ftg>)6HB$7PK^8)f6jN_6*Ip%hZjS67ZbZpb$KHk%{8|Ec_#y+kT(@e2B4_v z*caZ~m;j=7Cq3C`x^$4B#YD z?^|D+%i{(H3)Tl^yOmq1it90G8KR}`>l)_61C-H-H0?(ZSa&AH6}7sqM@bs!!wq`v zS11QBefDcyN+Nschoux;s$T%vfikD)?$$NZ)mK;*EEi&rHK@~oucx2=IcO*z>=0b* zyN^uQaK^`0VWYET+@9Rqg#v3Yy+(Sf3v2WQw~YjPFV8^DY8cY9P)Z%b$^Zg9OHueGwMXq!|{_)#x?YehlO`q zp2)gya{qv_3qPU1cO0wPnub2%>Q78tEaRCUa9hpV+(Sp9amCW)6@W-Z$%H%lMyxilRh4sp93ov_}$DT&?b)#xIzwT*% ze$0-&=O+`X*kTju1mK31`C&mj0zo8KYJY)Y)n?M>ZHMn^vKc9xj&Zg!K(HSE`&8PW zv|T<0VcdD4UAwtMv;!;uigyTAzN)yjDh>QBAaQV!vvzz z(r@q#04YF{1K|>|l_i>lRS9c$p$+tk@ByOho24ev^Bw#xCvD0)ITWgFdDxqE-D<5C zoR@w(e1}2@57B^b1I-YB99IM^6kBdS8>71cp?!VJd{_C6OPn1LZs5qgE!ioSD5@FT z*%_%t5tdXnT+6SKLdqSwv9|0p=rCGf{3(LiqbM8+$6{!Sk=|-CP~xXMLSo^cf_fQ6 zS8>k^Nd~{Vv(1)54MBbcq1XCpBm`utal|ijbL}d~*CKlEr_k2oL6Ou>5E=s zi!EVZPqJNfEe1vkG=WNc9615^`aEg&PSmy&g0BtrcJwA_O1;ev+sYTWASE)F{DhGd8sbx1vaYDk>^*1c-%Zde76>foA&yP z6~Mis^mlLnlfyDFhTjpiBjHj+DxwTVN9{?GnxMR$Pjq-A)=)# z@kof`X|#g(5z&LVOf1C;zfKYu>!&#cg*ab%*U8uz<` ze=XwPJXZESMn((|87e#8?C(fr%;SZ4$>&vaJ^e1GY7Xr!-77~Rc;Xk*D}Jk?Dd@1QxiPw56`5?jEn+p86^jzA8}ZP z3GSL|YtZ0`jA$HiN4RLFXu^L%31LPei>Jp zGYI~8cME`uLpO2`)5O?5l@sEN+{>n2jBO^-L68vj5mV4cHAPuXIKiq?uVS$&6YAsgoQ$7o%jXyW^Yigcyzs)PE9r=N96mM$Eh9;{o$CRiPBV;R2cktCGLQ(%@<5*ib&XV`Ri zpE;~OOEPgDvP@`;`0ofC9yzLh9)z2%+&|HN9BK ze~sJ1oLS)S?MhyMC@ez;XP1%mNSUSC*AS=JOLqDNo?%)WHFW;AFOE|ITMFOB&=x{; zlMp|nkVr~|a==9>S8@n))zM@Bfk%e6o_A{rf(DGyCkXhnU}i_2)5|yYG}r1t<@M=| z$-BV83(DXo_+TG9j3H8;LY`n4!|xisb9FnmmFw-2X+(qPch$^_TNYZSENGSP`1X%| zt7LfZ$uVSGV|m(VQNH48KMibK@R59Zkerr+h9tNjN!-O{%VlPeZ`To~_dEgc^}TA2 zHFjsoYlKC%qDS=*;&yiZ%%HeGa!_)nrF0I^9+qK#Ge~}veEI8hW6!;qSXm1D$ue~2 z-?|{B6ps0V!Mxe$OF8Zv^A>{eecLc_N+=2mde_Lv)HIHR=7xNgL@l$B2G*ijzG2_jP?H!G<@5WMy?7GU?LSi0*i-#Qh=5O39S z$TvNQ2gM{3Lm0}*-daXP<;W~Xu>wFrYFwXIoOUA+_ifEZrU`sC=)Qz?Nl$29R~DKN zFqvf_vp}$-lx^GCyc{HB3$w|JYB785$u}t@YYJ&oo!2U{Wh^aUoC>8#EbYp6IMxsvR@J-y2 z4I}u{1(F>4032i5x3{7?>0c#^b3GK3$O&n(D5W4FVWg0h)xuy66$??ojG}!IVrX1r z6iNFT5BqxvJ5E5*bp=_ykmF_1+0mHvn)3arr02hu8VxiXlD#ZE&-zlF2lfSgPF@gE zzFWmMyJp3jw|#c#F{;e1O}EQenZ-HCE!NHQzyof|t%r0eKGF1#JFDLloJe6qP6%oL z1KJzDQHkFw8}N&IT<(zDecCL2)X+cDMTkMbeEWOg%LHPzn0Gatg`=xyxeC!S2}O@& z>@*is7Pq(J?M{AD>PYvLs%?3ML_c`-4`w%ZXY-IqPxM`d=*5IPg=d?Sxeq{8GvhIs zH*tOiSJMMG;$maQi0%{dH8{Y3t%Zl_YrKbZQ9tf<$1)(NuW&d5$ELuCX_`?YQ-hlt zL_5MMMmn*G3J0#1Fg}u@+N!ng3*yVT92N|Iy?uYio8p?ZVq#YobDq^U6b( z@`L^q{*n)mkm4YD1vC>HjY#$2P-X13HF7pylp^Ff{M8Y#8@-0Z>c?wW#LvG>e) z8mJ%l+UCh9=my1C@@wW_;8N2T4YB&|yU5~Zh*i9q%DcF3` zydB%Ieeb)MRja4mLl&2KMHOasH=1Tptkzc~o)W#scK2}PmBoRau=iUc;splX$#+IS zudgG)Qa$RF)F24*=grV?JYlTkmKCKx4n|QjeU|Z|7Jcq*rzPq3xVpEQ6@>!qD|}ocbozXts2k~iU?9Ed*VO))HdYTkpC!z_+?zC% zT|yF912HaNEskLzz|kG;zvGtS+=b3!CO>Y`*|h9I=yb-;Cr-*Cm? zodqu+buICpLfZ76@JTcN^T;*9wMWDYMl+WA4b>^{BDr-k_xRHDRr63kh&<>xnXn(e z8FNd|ZI>9e%&&>9v{EW7?QzrJ&#(M*sa2J_b>ZcfnP{Mlt9FdC2D6;@V0U}g;&tk{ z7vp7SQsX!S2w&HVDg@hrB&6_zq4!t3BtiSBdnLg0Ui-r#<&72ENiZX`PP@5vbk%e2Cdl|TVpFD z)F0*QIIKi<-L@iFuY}jub{E?y>UBwPDim^}q7CeWu1y-;{5ZM*7`CR?@WXKBvl#S{ zT>Du1Kl?+#2AVbl$c?zeeSRo2jAM~NE|k12x{hU*&{Gw~BkS}wtH5`D74K)$zB z_jP^uVD~1heWve%L;QR~wS@00iXNvd_CJwTv^I8+y;wI~{fn|_rWv@QUC z8;OI$t}Yvn_Gry+F+a{#^qwcSq5?Wqs6#2^EJ#mM;?@6nc>>N3ggMjP3ln7gNI>Au=zuc5LS}xRJkd}1;4{* z6{jP$XQccAPK*x>*@>}kk=L276&t1tIGZaD7eCIk@P_!_%`R1!7SA5tnh(ZpAz8Lv z)bpM{iXTx>uVXph#F!?+63E{?bYXO%tP;)`aCI6S8T_bkvA%?L0ZWbegYL@1j(b3o zEaX`!hSe?dW}#+2sU(@9-1E(wP_jGSdL<{({5lA1qCW0b91uMiqG8_Q0-)uuiXLn% zG1^CBx*6jZ()~*c(KZ$6iIASEemMOrx9V3_8{OP(ms;U;Ih`(<+kw_nHXzG?0cJdU z_oD0P9H(nt1CB!;|Ab+i71f$W%)4NBwhGtZV;v&P>Qf!t%=o-j_I7$~5|Yyyb+5DQ zjH=fO3q2FqwdaR5Gvq0#eVC&xcnK{FZDYf8U`4MDSwHxd^aql+4Myab>(@P)mP$E< zY;vErB@8@2ESG(;_(az{c8Jv_z;|R#ylM;j9x29-FmD_I=`QtvWDP2(DdtC>EUv8& zC!Y^wh<-w2t-v0}@{bOE(o(L!>1po<BCtoI_H>y9YTbS6142TN-+~s4cGE}NIq)16+h;f*rnSS)a94K5__B41Zw@nH6{hk zJGEO{W=6|By8ZfHuS4w1td*la=$)+rEq*{lfA3Cg*hnSiS^|B*QWa&v526I*R6%S5 zf0R$T3};b2P@LSTQ1FWmx0^*S+nM7z_P;CpBRmh3wJT{V{nj(QK0YDT6At@dl4|!6 zsicX@&z`)N)DuRU)@-Jpjj^1ERt&RDFAz%RqNmR^jQFg43N!u{}qxJxPy zKWB9KAbg$Rr5MR3aO(5l47y}U^Q(lG#CTEe{SMh=I^|vd^_o4zUUj6n|8?Zx`e=#b zjh21tyf{@xcpd34scz=}5DEWz9vO36vDg94)bZBTBn3IElT!o(@0z&Z*zVgJ-9AD_ z=)`1OFal@25eyq#(j%qi3UZ4VHX=-Kv&5nR7<^{tn^Dk=7yu>@^~YR)4W6Dms12?tyL_dc0QmZ z4~3n~K;H6mJcjQZg`K6fKsqlMwLWl@qfV(lcpvYki#TIom;@lzNBb7cszdr=6u1iY z@4kNi9pNv-X1iY6Y>N$EPuOUEzQR$nE?-kszjPZ(0nbKe-o;3hW3bzMX4Vzy4&475!=B zo$>v;GdA6*q`O;i^`%=^{v@yeekm&KgWXpUzr~E?d8SOgu6^c%8-dgHUX%z^`l2tY zTU!0kBn5SZz;a?Qj}F-k?`$Jgp20xnp}uWQMVX8dzuG9U*hk(Md^+277z7W9wAed1 zr`N5+cE{9T^gL%MxJ=jjx_hmJx!zb^yZC`&B61Q)g^i%KW4e==2GXf?be1%VbmH>Q z;XISiuy~Te@1VczIiB#lQRb`e^t)ieBKO#x!Ic45xuNqp`=pr1f?Y+)1F%G|V2aCI zC%Cf3auCaL1b$oZrf{UjAQ%%f>`xWQx6k_Ws*z=jcFBr1dMA74C7dl=&PmZJ%JA=LN&=#8g;ue+_k*WG=Q zsULocxHxEf9>`t3Js^!_o3{s)jJej?&hflGgWmTzD-<6_=1cxGMbE%U`LmjQE^& zXcwe>Y74;Wkkokn-x@|%RR82_f@jH3N47sLkb;@?P@CQ z8*EebT8yUU)WAlbvd~=^9jZNE9p4-NG({lFs#!r^OeJnW7v~eL@mPnlU_nM?H%h(~ zQBCZgrHJ@==5lyo)|vTe8kq|=k;8_L77LQyEW0f6gFAE4_+dkF7~#s_$Ak>;wH`M#(B8L*(}yexA>9(-o2LPeNUHW8W|e7h2)aWiJt!yHs-o;>U)C)+c}KIfqYy9l!O3MUT4_ab&)>Z?Cc{ zCwtpRgYGw7Z9MAh0+mxs$M#Wq#u>Y6l{j{#>~6n%8=HRAhzF2os`5Eh$y!Y=Uok*o zV4Sg1xk}i;I|5W}Xhn5b((BDJqm9e@FDcI1cEEUJ_y3RfuKXS9wT%-AMIt+8-;X_H zE3)sq?0Z?0eaTiaHD!sxWUMu08BA(Q7)zEgitI}=M!wwNcthI6 z1i&H7#$f4@zV&+QoEy~xwcgVBu{#g7Uf-7!tg*lLJ4t9yz;AC@k8OXuzSR_r|DgYF z1Y){&W?H#Qm$8o)jV59Z99C#aug)0S^X^oXDHhL3Dd{biB`Z%+@v*O3C1W?E zJ$WD8$yLJKJ)dsA^6^KN0h~2rYIo=_?FrqK0B6aX;P0w_GobA0nld+irC6DF@1Y8MP;H)5sy( zu6`!vC_9^7S&mM5L8#H}88dD7Qo2o4;0W2YSw?_nMdg;g(dO z%PXqNhMXMNb&weF>j>JKCzB(;X7@~uOr%Ju_#)bCY!^>%?V2$U!JT&(`*BEIoilh2 z+Anp0ElE!YBbs|Impu>H;HGgLSOzBTPF4DJ{B1HbU%UD)Y+KiAiy(8l{Y-HyK@HL^ zq3V3|JjWpybN}%z`Q#r5#Mm<7BJkpvZVRNi)T=^P_aFQ-C-P zi+uNDBXlSa@u=1(TvL7ul=3Gq#&fsY?hBWE>cX+YqeFBD@QS#Y%Uj;~d) znv*Lw?H&Nsu%i^{FGvje_(~Vp#T^9~QlHy5<{&WPO59#<+tA8F84Sqv-6hVaOb)1g zOvUtrRClv%hWbbEY9o>ad<=;Ovjwdig{#>INQ=`a=O7&^EksT)69yxWWoTyiP+h-57cN)%S+aFxgXxD-IHH%g z&8)boA+R>kvsOgl^m%t=USBhUFoH=HIY4ElUZ*bvmCzm?p_~zG_!SQt?sa@&aP7%oH)eC%r7=I}Up7dnN zZ-Q#FIx0<8Ep8Kex~8wF;Y&cY4S!8I#%trubUbOeI306*2!O(h z{p{Vq^t@aQc|Ipv*1Y8{FmgOgr*eg(Zhm-TYew|0l)sZb`X+`pna|z+#nL1pD#zKf zmF+jjY)!XNHeUo;CG^nO-Y;jVdh1BWLf)&wrCkuu?b)p?_NpBn9(v`#y09@DSy-5e zDKYETb91ssW*5`~I7>6;C@izJQ!V>X&})^3oMFhU$eERh0v#_WBj907917G^xGvyo z3&igBxfS8^05-7EDxqg-nRhk%7Tl~Nf9=8Xc(Ix$qXcpE9{zD>g8$&v`NoEk5!JJ<=6#rQ8v@tYnht@OGoIr z-QeFq^SSl&W%(z+JLyq!3L7wSC{mu~>ZvIIZhv6QIhwyc z`hL|074r?`MU`p~myMC21I<5aqXzUf!Ea4$8g zzMsdup9u7z>WPS@NcHrPWsuDqpUNkoF3@(mW+AO_w)lum7&5txisuKoZD6d_NLN)HiwwZ{42<6Y`=myr=uPX~%1v*m*&D z63Sd^xCfXWD{LBem&_9p)fSewHKK|cp79Svh+88tt%NGnL(sp9yPKrm7r02b8%$?ayBOoH4{)yhpq zRk0AptIMOfb2S`e8z@#4i4D-DkUJJ2vp8w3LI2DdlL0A0ll{70Fnsl$4nUNY|K=DQ ze!-~XOJi?BOtFIX%lAVk6gu2~`rF#A_G-{BZ0d-{evOY% zZMJx)ado~;m{+E_6o1jhfK)!}Lfoxtk(6plI4U$Ov?>gHkEa|($JcJDmEhTnA78Tl z9KvwOD=(=%$`GEVaKr8enZ0%pjP8KPENrGfdQ3dZCS@=qycI|0e8`y~Wx7?#{@bui zP=g?FdTGwi^gD#*7EWDmn?H%pN&EtUphS$Psmz;&m4S-68*dw zkM=zbn~4(kvMuMiNDjdp{gD>(s`$t4q^8E)nGHh_WGEbx7nqX8`8ciYFDSPf*3a&~ zzN~5e?tClBhIW#JK+gScVU${=Oh8n@Jh=M_(Ef8^X_96p7glH%-`OMRj zWrF#ttsz9J=1S(!c9ngz#1!~A7G!=qw}f+5OMsO_3}U6VLWwoqhK5R90ns)C$h}~i zm1d^?0S_Cm=$hN&^SMq1xyWC<4I`C=^to5C>IKIyJF!HP=b8`rUvPT9_!XpSMNo)N z=Lmn8`{;vp|HC;`L`&JA_r|$W&`!5$8&{U(MrAT7QYGbN?mX>vCTa{(2L+p#;uCDN zb6!rgp^2@zF<7j*Tu>{XK^&`@f|X~Bj0$_ri9a%w>E#{uS&m#D?Ib1Bv1G|6%iEAj z!MrQ!O1FVwkcWr$D#Uc-*4uERrq#Bb2Ja(J0Vi`VZ_E`UuT9E^&6+XtP05&gsBD__ zMehNH`7JBc{92=tYu`kP1==5JNQh1K-y>)3UWs05Y;{bLSRjrJ-`xdii2Qxc#RM^) zeAg;NCJtk2?Ot8E><;)`%ADWi(ZHf-Jmn|fYpXusfo~SffF*X!ky=g=J;X9}1(Z&ZoGNYW@a5`c33(FA^sSS*jv^r41 zrf?cgHda@M4Oc{uoe*2f!SJ=5nXR}2+b(Mq^SpZ5F8|rNy?*PZcsmGn^G8Fs!KOli z#YNurcld9SLfzI&?nB5pD2hJ5^akMz{BN^`o5G3SDkB0X zKU0Z&mCU%5M)}=6jUPd4mg#D`;9lMPkth7l;3bL|;qu?Vsl&x|{i2^7%C5%9F@qsv z0T_T_;#oU?6CUP)PNN)Xnwv<`W4U#lAdC4ShKlmZOI{xNn=;$A%5{{(ov)~nb)7K( z$yA2cF4rBAq9h}uE|!lXjCZFhASyKVyR*j=eLcJ0cESM2@2$~`E}79ZLS0_OY0u+v zENBzixUJUAWM#zN^An|@ zhJIs7ce_Uh|pslcIZx`;<~>k3-_MM%ABK{<&m?S z>6~bKWl#&g1x32&sXT;3dDTp?(QTcO+jA3I3WatjxV7J$(C6{+6of)~{Y`lsdOFiL z3!iKGFmXsydwx3nG;mU$MePz|B7{HY_^>*327c-FeUA@a(t=9C@(6F_z=3`-@qwZe z(H%HD@zM(Fj4zz10$OTjL)>}GJ!sEU9AZkMpys0hU+~{QWna;eZ#zc;>0V*DC)&DI zJC5Uy$+`Crm;dpc0Oq0IX;2N2ODlgi+P6&esQ)LhhRP^O_fJ#B+(@9N}2frEo zkGKEvIAk$+J0%`jcPV-#_o2MhEAAB0Id&P>Ku~ys&7{ot!MQ(+mtwBYYKH;W|Izr@ zKc)BwF=bhdR%c1Ant~2jNfM zUmki22+Q+8#PU2OQOp9Tg5QwP)5yGetYq_Xh%>e*pw@Q9~1eWW!4nUIP#=NN;2Q^Z9=x zz*{=7=!Nc^=sCdKp)4=!U)*sfV2p$F!!)LYQSCm!)Vu=oWTY1Wuqsl9;ms%AyKH%2 zu5uR^glCGzaOpFk6}Vw=1{4X&SOLW3)64mQL*_O1YnRL^gq#A?-|djU2qn8hEvufs zMvl5}y#e``09RrbTqse#ws}!kMjmu9U_prT&1($pR=iJ!gSIz5Rcc+#<2GMF$=)2v)loYsGD?7xT&ejdvOTkIvVJmM!`NWSziSmO&J+?>?^q%(g$n#P!-iTuuVJb00I?9l&G$3Gr@Qy6%%kN5pRhT@+!_m4|2XaNZ2dtgZ+Z9Tv88Ji#nHqn=?@Y4ka z2m1w}*W8B$e?{Q8U;}u`u=M|Q8Nme%;7kgnO Date: Sat, 1 Dec 2018 17:39:54 +0300 Subject: [PATCH 161/233] update the jupyter notebook for generating the image --- examples/profile/results.ipynb | 138 ++++++++++++++++----------------- 1 file changed, 66 insertions(+), 72 deletions(-) diff --git a/examples/profile/results.ipynb b/examples/profile/results.ipynb index 1d944aa5..39bd08ff 100644 --- a/examples/profile/results.ipynb +++ b/examples/profile/results.ipynb @@ -2,10 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true - }, + "execution_count": 3, + "metadata": {}, "outputs": [], "source": [ "import pickle\n", @@ -14,10 +12,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, + "execution_count": 23, + "metadata": {}, "outputs": [], "source": [ "def pickle_load(path):\n", @@ -35,9 +31,9 @@ " for op in op_list:\n", " if op == 'gram':\n", " # Single element Gram matrix is flat inner.\n", - " curr_time = cpu['flatinner_time'] * 1000\n", + " curr_time = cpu['flatinner']['wall_time'] * 1000\n", " else:\n", - " curr_time = cpu[op + '_time'] * 1000\n", + " curr_time = cpu[op]['wall_time'] * 1000\n", " one_cpu.append(curr_time)\n", " d['one on CPU'] = one_cpu\n", " \n", @@ -45,27 +41,27 @@ " for op in op_list:\n", " if op == 'gram':\n", " # Single element Gram matrix is flat inner.\n", - " curr_time = gpu['flatinner_time'] * 1000\n", + " curr_time = gpu['flatinner']['wall_time'] * 1000\n", " else:\n", - " curr_time = gpu[op + '_time'] * 1000\n", + " curr_time = gpu[op]['wall_time'] * 1000\n", " one_gpu.append(curr_time)\n", " d['one on GPU'] = one_gpu\n", " \n", " batch_cpu = []\n", " for op in op_list:\n", " if op == 'gram':\n", - " curr_time = cpu['batch_gram_time'] * 1000 / 100**2\n", + " curr_time = cpu['batch_gram']['wall_time'] * 1000 / 100**2\n", " else:\n", - " curr_time = cpu['batch_' + op + '_time'] * 1000 / 100\n", + " curr_time = cpu['batch_' + op]['wall_time'] * 1000 / 100\n", " batch_cpu.append(curr_time)\n", " d['batch on CPU'] = batch_cpu\n", " \n", " batch_gpu = []\n", " for op in op_list:\n", " if op == 'gram':\n", - " curr_time = gpu['batch_gram_time'] * 1000 / 100.**2\n", + " curr_time = gpu['batch_gram']['wall_time'] * 1000 / 100.**2\n", " else:\n", - " curr_time = gpu['batch_' + op + '_time'] * 1000 / 100.\n", + " curr_time = gpu['batch_' + op]['wall_time'] * 1000 / 100.\n", "# curr_time = gpu['batch_' + op + '_time'] * 1000\n", " batch_gpu.append(curr_time)\n", " d['batch on GPU'] = batch_gpu\n", @@ -92,36 +88,34 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": true - }, + "execution_count": 24, + "metadata": {}, "outputs": [], "source": [ - "df = load('logs_cpu.pkl', 'logs_gpu.pkl', ttpy_path='logs_ttpy.pkl')" + "df = load('logs_cpu.pkl', 'fixed_logs_gpu.pkl', ttpy_path='logs_ttpy.pkl')" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", - "\n", "\n", " \n", @@ -138,50 +132,50 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
matvec11.1420.1290.1210.0030.0031.1900.7441.8850.140
matmul86.1910.1250.1330.0040.0049.8490.95017.4831.461
norm3.7901.9020.8930.4220.0502.1361.0190.2530.044
round73.0270.1590.1650.0060.00686.040165.9698.234161.102
gram0.1450.8060.7030.0290.6060.9730.0210.001
project_rank100116.8681.5641.6580.0170.0183.00113.2391.6450.226
\n", @@ -189,23 +183,23 @@ ], "text/plain": [ " ttpy, one on CPU one on CPU one on GPU batch on CPU \\\n", - "matvec 11.142 0.129 0.121 0.003 \n", - "matmul 86.191 0.125 0.133 0.004 \n", - "norm 3.790 1.902 0.893 0.422 \n", - "round 73.027 0.159 0.165 0.006 \n", - "gram 0.145 0.806 0.703 0.029 \n", - "project_rank100 116.868 1.564 1.658 0.017 \n", + "matvec 11.142 1.190 0.744 1.885 \n", + "matmul 86.191 9.849 0.950 17.483 \n", + "norm 3.790 2.136 1.019 0.253 \n", + "round 73.027 86.040 165.969 8.234 \n", + "gram 0.145 0.606 0.973 0.021 \n", + "project_rank100 116.868 3.001 13.239 1.645 \n", "\n", " batch on GPU \n", - "matvec 0.003 \n", - "matmul 0.004 \n", - "norm 0.050 \n", - "round 0.006 \n", + "matvec 0.140 \n", + "matmul 1.461 \n", + "norm 0.044 \n", + "round 161.102 \n", "gram 0.001 \n", - "project_rank100 0.018 " + "project_rank100 0.226 " ] }, - "execution_count": 7, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -216,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -227,12 +221,12 @@ "\\toprule\n", "{} & ttpy, one on CPU & one on CPU & one on GPU & batch on CPU & batch on GPU \\\\\n", "\\midrule\n", - "matvec & 11.142 & 0.129 & 0.121 & 0.003 & 0.003 \\\\\n", - "matmul & 86.191 & 0.125 & 0.133 & 0.004 & 0.004 \\\\\n", - "norm & 3.790 & 1.902 & 0.893 & 0.422 & 0.050 \\\\\n", - "round & 73.027 & 0.159 & 0.165 & 0.006 & 0.006 \\\\\n", - "gram & 0.145 & 0.806 & 0.703 & 0.029 & 0.001 \\\\\n", - "project\\_rank100 & 116.868 & 1.564 & 1.658 & 0.017 & 0.018 \\\\\n", + "matvec & 11.142 & 1.190 & 0.744 & 1.885 & 0.140 \\\\\n", + "matmul & 86.191 & 9.849 & 0.950 & 17.483 & 1.461 \\\\\n", + "norm & 3.790 & 2.136 & 1.019 & 0.253 & 0.044 \\\\\n", + "round & 73.027 & 86.040 & 165.969 & 8.234 & 161.102 \\\\\n", + "gram & 0.145 & 0.606 & 0.973 & 0.021 & 0.001 \\\\\n", + "project\\_rank100 & 116.868 & 3.001 & 13.239 & 1.645 & 0.226 \\\\\n", "\\bottomrule\n", "\\end{tabular}\n", "\n" @@ -269,7 +263,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.4" + "version": "3.5.5" } }, "nbformat": 4, From 5d1098ad2e109788ba4996fcb4858171c7508f08 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 8 Dec 2018 13:42:24 -0500 Subject: [PATCH 162/233] s/None/tf.no_op()/ None cannot be used in controlled dependencies --- t3f/autodiff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/autodiff.py b/t3f/autodiff.py index 96bd9aac..a93b3e20 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -100,7 +100,7 @@ def gradients(func, x, name='t3f_gradients', runtime_check=True): if runtime_check: assert_op = _is_invariant_to_input_transforms(function_value, func(x)) else: - assert_op = None + assert_op = tf.no_op() with tf.control_dependencies([assert_op]): cores_grad = tf.gradients(function_value, deltas) deltas = _enforce_gauge_conditions(cores_grad, left) @@ -162,7 +162,7 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', if runtime_check: assert_op = _is_invariant_to_input_transforms(function_value, func(x)) else: - assert_op = None + assert_op = tf.no_op() with tf.control_dependencies([assert_op]): vector_projected = riemannian.project(vector, x) cores_grad = tf.gradients(function_value, deltas) From b32f41878c167c4d056420cea0b97ca0f63c13b8 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 14:23:37 +0000 Subject: [PATCH 163/233] tmp benchmarkconfig implementation --- examples/profile/profile.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/profile/profile.py b/examples/profile/profile.py index f2c3cc0c..e9145af5 100644 --- a/examples/profile/profile.py +++ b/examples/profile/profile.py @@ -3,6 +3,26 @@ import pickle import argparse import tensorflow as tf + + +# TODO: remove this after the next release of TF (which should include +# tf.test.benchmark_config()) +try: + tf.test.benchmark_config() +except AttributeError: + from tensorflow import core + def benchmark_config(): + """Returns a tf.ConfigProto for disabling the dependency optimizer. + Returns: + A TensorFlow ConfigProto object. + """ + config = core.protobuf.config_pb2.ConfigProto() + config.graph_options.rewrite_options.dependency_optimization = ( + core.protobuf.rewriter_config_pb2.RewriterConfig.OFF) + return config + tf.test.benchmark_config = benchmark_config + + from tensorflow.python.client import device_lib import t3f From 550d3d3c1252af6f1599bf66ebc5277693545eb3 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 14:30:58 +0000 Subject: [PATCH 164/233] typo fix --- docs/tutorials/tensor_nets.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index 6d90a940..b3aab11e 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -65,7 +65,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Using TT-Matrices we can compactly represent densely connected layers in neural networks, which allows us to greatly reduce number of parameters. Matrix multiplication can be handled by the `t3f.matmul` method which allows for multiplying dense (ordinary) matrices and TT-Matrices. Very simple neural network could look as following (for initialization several options such as `t3f.initializers.glorot`, `t3f.initializers.he` or `t3f.random_matrix` are available):" + "Using TT-Matrices we can compactly represent densely connected layers in neural networks, which allows us to greatly reduce number of parameters. Matrix multiplication can be handled by the `t3f.matmul` method which allows for multiplying dense (ordinary) matrices and TT-Matrices. Very simple neural network could look as following (for initialization several options such as `t3f.glorot_initializer`, `t3f.he_initializer` or `t3f.random_matrix` are available):" ] }, { From 05a02e010be00fb330d9c8ebaf4d657a2ee19ff2 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 14:33:04 +0000 Subject: [PATCH 165/233] init moving neural utils to the main library --- examples/utils/tt_dense.py => t3f/neural.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/utils/tt_dense.py => t3f/neural.py (100%) diff --git a/examples/utils/tt_dense.py b/t3f/neural.py similarity index 100% rename from examples/utils/tt_dense.py rename to t3f/neural.py From 621c4681c1e969cd1430d382786ee41b41c30c31 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 14:34:23 +0000 Subject: [PATCH 166/233] better naming and init docstring --- t3f/neural.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/t3f/neural.py b/t3f/neural.py index de916eee..90f10c36 100644 --- a/t3f/neural.py +++ b/t3f/neural.py @@ -1,3 +1,5 @@ +"""Utils for simplifying building neural networks with TT-layers""" + from keras.engine.topology import Layer from keras.layers import Activation import t3f @@ -8,7 +10,7 @@ inits = ['glorot', 'he', 'lecun'] -class TTDense(Layer): +class KerasTTDense(Layer): counter = 0 def __init__(self, row_dims, column_dims, tt_rank=2, init='glorot', @@ -41,7 +43,7 @@ def __init__(self, row_dims, column_dims, tt_rank=2, init='glorot', self.bias = bias self.bias_init = bias_init self.init = init - super(TTDense, self).__init__(**kwargs) + super(Keras, self).__init__(**kwargs) def build(self, input_shape): if self.init == 'glorot': @@ -56,15 +58,15 @@ def build(self, input_shape): else: raise ValueError('Unknown init "%s", only %s are supported' % (self.init, inits)) - name = 'tt_dense_matrix_{}'.format(TTDense.counter) + name = 'tt_dense_matrix_{}'.format(KerasTTDense.counter) self.W = t3f.get_variable(name, initializer=initializer) self.b = None if self.bias: - b_name = 'tt_dense_b_{}'.format(TTDense.counter) + b_name = 'tt_dense_b_{}'.format(KerasTTDense.counter) b_init = tf.constant_initializer(self.bias_init) self.b = tf.get_variable(b_name, shape=self.output_dim, initializer=b_init) - TTDense.counter += 1 + KerasTTDense.counter += 1 self.trainable_weights = list(self.W.tt_cores) if self.b is not None: self.trainable_weights.append(self.b) From 5a17deb18da86fc65477d9d165db828e06bf3870 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 14:44:18 +0000 Subject: [PATCH 167/233] better argument names and other NITs --- t3f/neural.py | 48 +++++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/t3f/neural.py b/t3f/neural.py index 90f10c36..938d6f30 100644 --- a/t3f/neural.py +++ b/t3f/neural.py @@ -6,27 +6,25 @@ import tensorflow as tf import numpy as np -activations = ['relu', 'sigmoid', 'tanh', 'softmax'] -inits = ['glorot', 'he', 'lecun'] - class KerasTTDense(Layer): counter = 0 - def __init__(self, row_dims, column_dims, tt_rank=2, init='glorot', - activation='relu', bias=True, bias_init=0.1, **kwargs): - """Creates a TT-Matrix based Dense layer. + def __init__(self, input_dims, output_dims, tt_rank=2, + activation='relu', use_bias=True, kernel_initializer='glorot', + bias_initializer=0.1, **kwargs): + """Creates a TT-Matrix based Dense Keras layer. Args: - row_dims: an array, shape of the matrix row index - column_dims: an array, shape of the matrix column index + input_dims: an array, tensor shape of the matrix row index + ouput_dims: an array, tensor shape of the matrix column index tt_rank: a number or an array, desired tt-rank of the TT-Matrix - init: string specifying initializer for the TT-Matrix. Possible - values are 'glorot', 'he', 'lecun'. activation: string, specifies the activation function. Possible values are 'relu', 'sigmoid', 'tanh', 'softmax' and None - bias: bool, whether to use bias - bias_init: a number, initialization value of the bias + use_bias: bool, whether to use bias + kernel_initializer: string specifying initializer for the TT-Matrix. + Possible values are 'glorot', 'he', and 'lecun'. + bias_initializer: a number, initialization value of the bias Returns: Layer object corresponding to multiplication by a TT-Matrix @@ -34,16 +32,17 @@ def __init__(self, row_dims, column_dims, tt_rank=2, init='glorot', an elementwise activation Raises: - ValueError if the provided activation or init is unknown + ValueError if the provided activation or kernel_initializer is + unknown. """ - self.tt_shape = [row_dims, column_dims] - self.output_dim = np.prod(column_dims) + self.tt_shape = [input_dims, output_dims] + self.output_dim = np.prod(output_dims) self.tt_rank = tt_rank self.activation = activation - self.bias = bias - self.bias_init = bias_init - self.init = init - super(Keras, self).__init__(**kwargs) + self.use_bias = use_bias + self.kernel_initializer = kernel_initializer + self.bias_initializer = bias_initializer + super(KerasTTDense, self).__init__(**kwargs) def build(self, input_shape): if self.init == 'glorot': @@ -56,8 +55,8 @@ def build(self, input_shape): initializer = t3f.lecun_initializer(self.tt_shape, tt_rank=self.tt_rank) else: - raise ValueError('Unknown init "%s", only %s are supported' - % (self.init, inits)) + raise ValueError('Unknown kernel_initializer "%s", only "glorot",' + '"he", and "lecun" are supported' % self.init) name = 'tt_dense_matrix_{}'.format(KerasTTDense.counter) self.W = t3f.get_variable(name, initializer=initializer) self.b = None @@ -77,12 +76,7 @@ def call(self, x): else: h = t3f.matmul(x, self.W) if self.activation is not None: - if self.activation in activations: - h = Activation(self.activation)(h) - else: - raise ValueError('Unknown activation "%s", only %s and None ' - 'are supported' - % (self.activation, activations)) + h = Activation(self.activation)(h) return h def compute_output_shape(self, input_shape): From ebb1d519c47772f71b0264e8f9b3ab77cd32a023 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 14:56:54 +0000 Subject: [PATCH 168/233] fix identation --- t3f/neural.py | 134 +++++++++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/t3f/neural.py b/t3f/neural.py index 938d6f30..8d398e90 100644 --- a/t3f/neural.py +++ b/t3f/neural.py @@ -1,83 +1,83 @@ """Utils for simplifying building neural networks with TT-layers""" +import numpy as np from keras.engine.topology import Layer from keras.layers import Activation import t3f import tensorflow as tf -import numpy as np class KerasTTDense(Layer): - counter = 0 + counter = 0 - def __init__(self, input_dims, output_dims, tt_rank=2, - activation='relu', use_bias=True, kernel_initializer='glorot', - bias_initializer=0.1, **kwargs): - """Creates a TT-Matrix based Dense Keras layer. + def __init__(self, input_dims, output_dims, tt_rank=2, + activation='relu', use_bias=True, kernel_initializer='glorot', + bias_initializer=0.1, **kwargs): + """Creates a TT-Matrix based Dense Keras layer. - Args: - input_dims: an array, tensor shape of the matrix row index - ouput_dims: an array, tensor shape of the matrix column index - tt_rank: a number or an array, desired tt-rank of the TT-Matrix - activation: string, specifies the activation function. Possible - values are 'relu', 'sigmoid', 'tanh', 'softmax' and None - use_bias: bool, whether to use bias - kernel_initializer: string specifying initializer for the TT-Matrix. - Possible values are 'glorot', 'he', and 'lecun'. - bias_initializer: a number, initialization value of the bias + Args: + input_dims: an array, tensor shape of the matrix row index + ouput_dims: an array, tensor shape of the matrix column index + tt_rank: a number or an array, desired tt-rank of the TT-Matrix + activation: string, specifies the activation function. Possible + values are 'relu', 'sigmoid', 'tanh', 'softmax' and None + use_bias: bool, whether to use bias + kernel_initializer: string specifying initializer for the TT-Matrix. + Possible values are 'glorot', 'he', and 'lecun'. + bias_initializer: a number, initialization value of the bias - Returns: - Layer object corresponding to multiplication by a TT-Matrix - followed by addition of a bias and applying - an elementwise activation + Returns: + Layer object corresponding to multiplication by a TT-Matrix + followed by addition of a bias and applying + an elementwise activation - Raises: - ValueError if the provided activation or kernel_initializer is - unknown. - """ - self.tt_shape = [input_dims, output_dims] - self.output_dim = np.prod(output_dims) - self.tt_rank = tt_rank - self.activation = activation - self.use_bias = use_bias - self.kernel_initializer = kernel_initializer - self.bias_initializer = bias_initializer - super(KerasTTDense, self).__init__(**kwargs) + Raises: + ValueError if the provided activation or kernel_initializer is + unknown. + """ + self.tt_shape = [input_dims, output_dims] + self.output_dim = np.prod(output_dims) + self.tt_rank = tt_rank + self.activation = activation + self.use_bias = use_bias + self.kernel_initializer = kernel_initializer + self.bias_initializer = bias_initializer + super(KerasTTDense, self).__init__(**kwargs) - def build(self, input_shape): - if self.init == 'glorot': - initializer = t3f.glorot_initializer(self.tt_shape, - tt_rank=self.tt_rank) - elif self.init == 'he': - initializer = t3f.he_initializer(self.tt_shape, - tt_rank=self.tt_rank) - elif self.init == 'lecun': - initializer = t3f.lecun_initializer(self.tt_shape, - tt_rank=self.tt_rank) - else: - raise ValueError('Unknown kernel_initializer "%s", only "glorot",' - '"he", and "lecun" are supported' % self.init) - name = 'tt_dense_matrix_{}'.format(KerasTTDense.counter) - self.W = t3f.get_variable(name, initializer=initializer) - self.b = None - if self.bias: - b_name = 'tt_dense_b_{}'.format(KerasTTDense.counter) - b_init = tf.constant_initializer(self.bias_init) - self.b = tf.get_variable(b_name, shape=self.output_dim, - initializer=b_init) - KerasTTDense.counter += 1 - self.trainable_weights = list(self.W.tt_cores) - if self.b is not None: - self.trainable_weights.append(self.b) + def build(self, input_shape): + if self.init == 'glorot': + initializer = t3f.glorot_initializer(self.tt_shape, + tt_rank=self.tt_rank) + elif self.init == 'he': + initializer = t3f.he_initializer(self.tt_shape, + tt_rank=self.tt_rank) + elif self.init == 'lecun': + initializer = t3f.lecun_initializer(self.tt_shape, + tt_rank=self.tt_rank) + else: + raise ValueError('Unknown kernel_initializer "%s", only "glorot",' + '"he", and "lecun" are supported' % self.init) + name = 'tt_dense_matrix_{}'.format(KerasTTDense.counter) + self.W = t3f.get_variable(name, initializer=initializer) + self.b = None + if self.bias: + b_name = 'tt_dense_b_{}'.format(KerasTTDense.counter) + b_init = tf.constant_initializer(self.bias_init) + self.b = tf.get_variable(b_name, shape=self.output_dim, + initializer=b_init) + KerasTTDense.counter += 1 + self.trainable_weights = list(self.W.tt_cores) + if self.b is not None: + self.trainable_weights.append(self.b) - def call(self, x): - if self.bias: - h = t3f.matmul(x, self.W) + self.b - else: - h = t3f.matmul(x, self.W) - if self.activation is not None: - h = Activation(self.activation)(h) - return h + def call(self, x): + if self.bias: + h = t3f.matmul(x, self.W) + self.b + else: + h = t3f.matmul(x, self.W) + if self.activation is not None: + h = Activation(self.activation)(h) + return h - def compute_output_shape(self, input_shape): - return (input_shape[0], self.output_dim) + def compute_output_shape(self, input_shape): + return (input_shape[0], self.output_dim) From 0f850237a854407512bfd9abdaf63410b8c39719 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:09:34 +0000 Subject: [PATCH 169/233] simplest possible test: that it compiles --- t3f/neural_test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 t3f/neural_test.py diff --git a/t3f/neural_test.py b/t3f/neural_test.py new file mode 100644 index 00000000..7c075dd7 --- /dev/null +++ b/t3f/neural_test.py @@ -0,0 +1,23 @@ +import numpy as np +import tensorflow as tf + +from t3f import neural + + +class _NeuralTest(): + + def testKerasDense(self): + layer = neural.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5]) + layer(tf.random_normal((20, 28*28))) + + +class NeuralTestFloat32(tf.test.TestCase, _NeuralTest): + dtype = tf.float32 + + +class NeuralTestFloat64(tf.test.TestCase, _NeuralTest): + dtype = tf.float64 + + +if __name__ == "__main__": + tf.test.main() From 560c6c3ee341a70aa60f523f693a180d3dedddb3 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:10:07 +0000 Subject: [PATCH 170/233] KerasTTDense -> KerasDense and fixes w.r.t. new arg names --- t3f/neural.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/t3f/neural.py b/t3f/neural.py index 8d398e90..2a28eb1c 100644 --- a/t3f/neural.py +++ b/t3f/neural.py @@ -7,7 +7,7 @@ import tensorflow as tf -class KerasTTDense(Layer): +class KerasDense(Layer): counter = 0 def __init__(self, input_dims, output_dims, tt_rank=2, @@ -42,36 +42,36 @@ def __init__(self, input_dims, output_dims, tt_rank=2, self.use_bias = use_bias self.kernel_initializer = kernel_initializer self.bias_initializer = bias_initializer - super(KerasTTDense, self).__init__(**kwargs) + super(KerasDense, self).__init__(**kwargs) def build(self, input_shape): - if self.init == 'glorot': + if self.kernel_initializer == 'glorot': initializer = t3f.glorot_initializer(self.tt_shape, tt_rank=self.tt_rank) - elif self.init == 'he': + elif self.kernel_initializer == 'he': initializer = t3f.he_initializer(self.tt_shape, tt_rank=self.tt_rank) - elif self.init == 'lecun': + elif self.kernel_initializer == 'lecun': initializer = t3f.lecun_initializer(self.tt_shape, tt_rank=self.tt_rank) else: raise ValueError('Unknown kernel_initializer "%s", only "glorot",' - '"he", and "lecun" are supported' % self.init) - name = 'tt_dense_matrix_{}'.format(KerasTTDense.counter) + '"he", and "lecun" are supported' % self.kernel_initializer) + name = 'tt_dense_matrix_{}'.format(KerasDense.counter) self.W = t3f.get_variable(name, initializer=initializer) self.b = None - if self.bias: - b_name = 'tt_dense_b_{}'.format(KerasTTDense.counter) - b_init = tf.constant_initializer(self.bias_init) + if self.use_bias: + b_name = 'tt_dense_b_{}'.format(KerasDense.counter) + b_init = tf.constant_initializer(self.bias_initializer) self.b = tf.get_variable(b_name, shape=self.output_dim, initializer=b_init) - KerasTTDense.counter += 1 + KerasDense.counter += 1 self.trainable_weights = list(self.W.tt_cores) if self.b is not None: self.trainable_weights.append(self.b) def call(self, x): - if self.bias: + if self.use_bias: h = t3f.matmul(x, self.W) + self.b else: h = t3f.matmul(x, self.W) From 6bf9899e7d6c334eabc21fb67dcc408aa8bfe055 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:10:49 +0000 Subject: [PATCH 171/233] slightly less code duplication --- t3f/neural.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/t3f/neural.py b/t3f/neural.py index 2a28eb1c..7196362c 100644 --- a/t3f/neural.py +++ b/t3f/neural.py @@ -71,13 +71,12 @@ def build(self, input_shape): self.trainable_weights.append(self.b) def call(self, x): + res = t3f.matmul(x, self.W) if self.use_bias: - h = t3f.matmul(x, self.W) + self.b - else: - h = t3f.matmul(x, self.W) + res += self.b if self.activation is not None: - h = Activation(self.activation)(h) - return h + res = Activation(self.activation)(res) + return res def compute_output_shape(self, input_shape): return (input_shape[0], self.output_dim) From 8e2602df303ccba902fd9ec45cffe773346f31a0 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:12:04 +0000 Subject: [PATCH 172/233] do not use activations by default (like in keras.dense) --- t3f/neural.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/t3f/neural.py b/t3f/neural.py index 7196362c..01781feb 100644 --- a/t3f/neural.py +++ b/t3f/neural.py @@ -11,7 +11,7 @@ class KerasDense(Layer): counter = 0 def __init__(self, input_dims, output_dims, tt_rank=2, - activation='relu', use_bias=True, kernel_initializer='glorot', + activation=None, use_bias=True, kernel_initializer='glorot', bias_initializer=0.1, **kwargs): """Creates a TT-Matrix based Dense Keras layer. @@ -19,8 +19,7 @@ def __init__(self, input_dims, output_dims, tt_rank=2, input_dims: an array, tensor shape of the matrix row index ouput_dims: an array, tensor shape of the matrix column index tt_rank: a number or an array, desired tt-rank of the TT-Matrix - activation: string, specifies the activation function. Possible - values are 'relu', 'sigmoid', 'tanh', 'softmax' and None + activation: [None] string or None, specifies the activation function. use_bias: bool, whether to use bias kernel_initializer: string specifying initializer for the TT-Matrix. Possible values are 'glorot', 'he', and 'lecun'. From 85caab8f4ee7888f8649af662166a6dd3f784e5a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:35:56 +0000 Subject: [PATCH 173/233] better naming for tf.variables --- t3f/neural.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/t3f/neural.py b/t3f/neural.py index 01781feb..c59bdb31 100644 --- a/t3f/neural.py +++ b/t3f/neural.py @@ -1,5 +1,6 @@ """Utils for simplifying building neural networks with TT-layers""" +from itertools import count import numpy as np from keras.engine.topology import Layer from keras.layers import Activation @@ -8,7 +9,7 @@ class KerasDense(Layer): - counter = 0 + _counter = count(0) def __init__(self, input_dims, output_dims, tt_rank=2, activation=None, use_bias=True, kernel_initializer='glorot', @@ -34,6 +35,7 @@ def __init__(self, input_dims, output_dims, tt_rank=2, ValueError if the provided activation or kernel_initializer is unknown. """ + self.counter = next(self._counter) self.tt_shape = [input_dims, output_dims] self.output_dim = np.prod(output_dims) self.tt_rank = tt_rank @@ -55,22 +57,22 @@ def build(self, input_shape): tt_rank=self.tt_rank) else: raise ValueError('Unknown kernel_initializer "%s", only "glorot",' - '"he", and "lecun" are supported' % self.kernel_initializer) - name = 'tt_dense_matrix_{}'.format(KerasDense.counter) - self.W = t3f.get_variable(name, initializer=initializer) - self.b = None - if self.use_bias: - b_name = 'tt_dense_b_{}'.format(KerasDense.counter) - b_init = tf.constant_initializer(self.bias_initializer) - self.b = tf.get_variable(b_name, shape=self.output_dim, - initializer=b_init) - KerasDense.counter += 1 - self.trainable_weights = list(self.W.tt_cores) + '"he", and "lecun" are supported' + % self.kernel_initializer) + name = 'tt_dense_{}'.format(self.counter) + with tf.variable_scope(name): + self.matrix = t3f.get_variable('matrix', initializer=initializer) + self.b = None + if self.use_bias: + b_init = tf.constant_initializer(self.bias_initializer) + self.b = tf.get_variable('bias', shape=self.output_dim, + initializer=b_init) + self.trainable_weights = list(self.matrix.tt_cores) if self.b is not None: self.trainable_weights.append(self.b) def call(self, x): - res = t3f.matmul(x, self.W) + res = t3f.matmul(x, self.matrix) if self.use_bias: res += self.b if self.activation is not None: From 611ef6cd62da2535b372613c72fbe1632a930f5f Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:39:05 +0000 Subject: [PATCH 174/233] test creating variables with new name --- t3f/neural_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/t3f/neural_test.py b/t3f/neural_test.py index 7c075dd7..9978fa9d 100644 --- a/t3f/neural_test.py +++ b/t3f/neural_test.py @@ -7,8 +7,13 @@ class _NeuralTest(): def testKerasDense(self): + # Try to create the layer twice to check that it won't crush saying the + # variable already exist. + x = tf.random_normal((20, 28*28)) layer = neural.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5]) - layer(tf.random_normal((20, 28*28))) + layer(x) + layer = neural.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5]) + layer(x) class NeuralTestFloat32(tf.test.TestCase, _NeuralTest): From 6dcb8d0e63ea08c538ea8c773a7a87843eab0952 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:50:02 +0000 Subject: [PATCH 175/233] import neural module --- t3f/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/t3f/__init__.py b/t3f/__init__.py index 0b497678..4f1cb7c1 100644 --- a/t3f/__init__.py +++ b/t3f/__init__.py @@ -75,6 +75,7 @@ import t3f.approximate as approximate import t3f.kronecker as kronecker +import t3f.neural as nn import t3f.utils as utils _directly_imported = ['tensor_train_base', 'tensor_train', 'tensor_train_batch', From b3a18c9c583e34931f42a7346adda1805beee2ab Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:55:53 +0000 Subject: [PATCH 176/233] neural -> nn --- t3f/{neural.py => nn.py} | 0 t3f/{neural_test.py => nn_test.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename t3f/{neural.py => nn.py} (100%) rename t3f/{neural_test.py => nn_test.py} (100%) diff --git a/t3f/neural.py b/t3f/nn.py similarity index 100% rename from t3f/neural.py rename to t3f/nn.py diff --git a/t3f/neural_test.py b/t3f/nn_test.py similarity index 100% rename from t3f/neural_test.py rename to t3f/nn_test.py From 4662e0c3fc17eaf0f6c42917e8137e54c46c2517 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:56:14 +0000 Subject: [PATCH 177/233] do not import as (it doesnt do anything) --- t3f/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/t3f/__init__.py b/t3f/__init__.py index 4f1cb7c1..05d318eb 100644 --- a/t3f/__init__.py +++ b/t3f/__init__.py @@ -73,10 +73,10 @@ from t3f.autodiff import gradients from t3f.autodiff import hessian_vector_product -import t3f.approximate as approximate -import t3f.kronecker as kronecker -import t3f.neural as nn -import t3f.utils as utils +import t3f.approximate +import t3f.kronecker +import t3f.nn +import t3f.utils _directly_imported = ['tensor_train_base', 'tensor_train', 'tensor_train_batch', 'variables', 'ops', 'batch_ops', 'initializers', From e3528b14b9ce65b8918ef9dd344b58285ab6ae3a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 15:57:54 +0000 Subject: [PATCH 178/233] fix: switch to new name --- t3f/nn_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t3f/nn_test.py b/t3f/nn_test.py index 9978fa9d..333a8b0a 100644 --- a/t3f/nn_test.py +++ b/t3f/nn_test.py @@ -1,7 +1,7 @@ import numpy as np import tensorflow as tf -from t3f import neural +from t3f import nn class _NeuralTest(): @@ -10,9 +10,9 @@ def testKerasDense(self): # Try to create the layer twice to check that it won't crush saying the # variable already exist. x = tf.random_normal((20, 28*28)) - layer = neural.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5]) + layer = nn.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5]) layer(x) - layer = neural.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5]) + layer = nn.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5]) layer(x) From d5221057067448742d0f74454d553f92c32f6eec Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 16:06:02 +0000 Subject: [PATCH 179/233] use t3f.nn --- docs/tutorials/tensor_nets.ipynb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index b3aab11e..2c96f7d2 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -8,6 +8,8 @@ "\n", "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", "\n", + "Also, if you run this in Colab, consider switching to GPU: **Runtime -> Change runtime type -> Hardware accelerator -> GPU**.\n", + "\n", "In this notebook we provide an example of how to build a simple Tensor Net (see https://arxiv.org/abs/1509.06569).\n", "\n", "The main ingredient is the so-called TT-Matrix, a generalization of the Kronecker product matrices, i.e. matrices of the form \n", @@ -160,7 +162,9 @@ "source": [ "model = Sequential()\n", "model.add(Flatten(input_shape=(28, 28)))\n", - "model.add(TTDense(row_dims=[7, 4, 7, 4], column_dims=[5, 5, 5, 5], tt_rank=4, init='glorot', activation='relu', bias_init=1e-3))\n", + "tt_layer = t3f.nn.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5],\n", + " tt_rank=4, activation='relu', bias_init=1e-3)\n", + "model.add(tt_layer)\n", "model.add(Dense(10))\n", "model.add(Activation('softmax'))" ] @@ -262,7 +266,7 @@ "collapsed": true }, "source": [ - "Let us now train an ordinary DNN (without TT-Matrices) and show how we can compress it using the TT decomposition." + "Let us now train an ordinary DNN (without TT-Matrices) and show how we can compress it using the TT decomposition. (In contrast to directly training a TT-layer from scratch in the example above.)" ] }, { @@ -425,7 +429,9 @@ "source": [ "model = Sequential()\n", "model.add(Flatten(input_shape=(28, 28)))\n", - "model.add(TTDense(row_dims=[7, 4, 7, 4], column_dims=[5, 5, 5, 5], tt_rank=16, activation='relu'))\n", + "tt_layer = t3f.nn.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5],\n", + " tt_rank=16, activation='relu')\n", + "model.add(tt_layer)\n", "model.add(Dense(10))\n", "model.add(Activation('softmax'))" ] From 585dae6afd896c5b320111f7946e0530046a9ce0 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 16:14:54 +0000 Subject: [PATCH 180/233] install keras for testing --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 08ad9e23..68cc66c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: # command to install dependencies install: # Python 2.7 has very old pip by default that doesn't have tensorflow. - - pip install numpy tensorflow==$TF_VERSION + - pip install numpy tensorflow==$TF_VERSION keras - pip install coveralls # command to run tests script: From c6999c3156089ea545a863beb79a4a8fdad9d250 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 16:25:57 +0000 Subject: [PATCH 181/233] remove outdated import --- docs/tutorials/tensor_nets.ipynb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index 2c96f7d2..1a8c5516 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -115,8 +115,7 @@ "from keras.layers import Dense, Activation, Dropout, Flatten\n", "import numpy as np\n", "from keras.utils import to_categorical\n", - "from keras import optimizers\n", - "from utils import TTDense" + "from keras import optimizers" ] }, { From 7be13cc23c3c894e3a97e646baa7c1fe01cd144b Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 16:31:25 +0000 Subject: [PATCH 182/233] fix argument name --- docs/tutorials/tensor_nets.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index 1a8c5516..af14385f 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -162,7 +162,8 @@ "model = Sequential()\n", "model.add(Flatten(input_shape=(28, 28)))\n", "tt_layer = t3f.nn.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5],\n", - " tt_rank=4, activation='relu', bias_init=1e-3)\n", + " tt_rank=4, activation='relu',\n", + " bias_initializer=1e-3)\n", "model.add(tt_layer)\n", "model.add(Dense(10))\n", "model.add(Activation('softmax'))" From d1ec306d2a2736204bc54a04becb4c8c09135507 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 16:32:58 +0000 Subject: [PATCH 183/233] emphasize number of parameters --- docs/tutorials/tensor_nets.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index af14385f..08eb232b 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -559,7 +559,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We see that we were able to achieve higher validation accuracy than we had in the plain DNN, while keeping the number of parameters extremely small." + "We see that we were able to achieve higher validation accuracy than we had in the plain DNN, while keeping the number of parameters extremely small (21845 vs 496885 parameters in the uncompressed model)." ] }, { From 12485ace9adfc7caba9f542a117526dc1ca3b4e4 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 16:45:32 +0000 Subject: [PATCH 184/233] update notebook after running in colab (fresh outputs) --- docs/tutorials/tensor_nets.ipynb | 1343 +++++++++++++++++------------- 1 file changed, 753 insertions(+), 590 deletions(-) diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index 08eb232b..23cdcde8 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -1,596 +1,759 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Tensor Nets (compressing neural networks)\n", - "\n", - "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", - "\n", - "Also, if you run this in Colab, consider switching to GPU: **Runtime -> Change runtime type -> Hardware accelerator -> GPU**.\n", - "\n", - "In this notebook we provide an example of how to build a simple Tensor Net (see https://arxiv.org/abs/1509.06569).\n", - "\n", - "The main ingredient is the so-called TT-Matrix, a generalization of the Kronecker product matrices, i.e. matrices of the form \n", - "$$A = A_1 \\otimes A_2 \\cdots \\otimes A_n$$\n", - "\n", - "In `t3f` TT-Matrices are represented using the `TensorTrain` class." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import tensorflow as tf\n", - "import keras.backend as K\n", - "\n", - "tf.set_random_seed(0)\n", - "np.random.seed(0)\n", - "sess = tf.InteractiveSession()\n", - "K.set_session(sess)\n", - "\n", - "try:\n", - " import t3f\n", - "except ImportError:\n", - " # Install T3F if it's not already installed.\n", - " !git clone https://github.com/Bihaqo/t3f.git\n", - " !cd t3f; pip install .\n", - " import t3f" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "A TT-Matrix of size 784 x 625, underlying tensor shape: (4, 7, 4, 7) x (5, 5, 5, 5), TT-ranks: (1, 2, 2, 2, 1)\n" - ] - } - ], - "source": [ - "W = t3f.random_matrix([[4, 7, 4, 7], [5, 5, 5, 5]], tt_rank=2)\n", - "\n", - "print(W)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Using TT-Matrices we can compactly represent densely connected layers in neural networks, which allows us to greatly reduce number of parameters. Matrix multiplication can be handled by the `t3f.matmul` method which allows for multiplying dense (ordinary) matrices and TT-Matrices. Very simple neural network could look as following (for initialization several options such as `t3f.glorot_initializer`, `t3f.he_initializer` or `t3f.random_matrix` are available):" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "x = tf.placeholder(tf.float32, [None, 784])\n", - "y = tf.placeholder(tf.int64, [None])\n", - "\n", - "initializer = t3f.glorot_initializer([[4, 7, 4, 7], [5, 5, 5, 5]], tt_rank=2)\n", - "W1 = t3f.get_variable('W1', initializer=initializer) \n", - "b1 = tf.get_variable('b1', shape=[625])\n", - "h1 = t3f.matmul(x, W1) + b1\n", - "h1 = tf.nn.relu(h1)\n", - "\n", - "W2 = tf.get_variable('W2', shape=[625, 10])\n", - "b2 = tf.get_variable('b2', shape=[10])\n", - "h2 = tf.matmul(h1, W2) + b2\n", - "\n", - "y_ = tf.one_hot(y, 10)\n", - "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=h2))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For convenience we have implemented a layer analogous to *Keras* `Dense` layer but with a TT-Matrix instead of an ordinary matrix. An example of fully trainable net is provided below." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "from keras.datasets import mnist\n", - "from keras.models import Sequential\n", - "from keras.layers import Dense, Activation, Dropout, Flatten\n", - "import numpy as np\n", - "from keras.utils import to_categorical\n", - "from keras import optimizers" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "(x_train, y_train), (x_test, y_test) = mnist.load_data()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Some preprocessing..." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "x_train = x_train / 127.5 - 1.0\n", - "x_test = x_test / 127.5 - 1.0\n", - "\n", - "y_train = to_categorical(y_train, num_classes=10)\n", - "y_test = to_categorical(y_test, num_classes=10)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "model = Sequential()\n", - "model.add(Flatten(input_shape=(28, 28)))\n", - "tt_layer = t3f.nn.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5],\n", - " tt_rank=4, activation='relu',\n", - " bias_initializer=1e-3)\n", - "model.add(tt_layer)\n", - "model.add(Dense(10))\n", - "model.add(Activation('softmax'))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "_________________________________________________________________\n", - "Layer (type) Output Shape Param # \n", - "=================================================================\n", - "flatten_1 (Flatten) (None, 784) 0 \n", - "_________________________________________________________________\n", - "tt_dense_1 (TTDense) (None, 625) 1725 \n", - "_________________________________________________________________\n", - "dense_1 (Dense) (None, 10) 6260 \n", - "_________________________________________________________________\n", - "activation_2 (Activation) (None, 10) 0 \n", - "=================================================================\n", - "Total params: 7,985\n", - "Trainable params: 7,985\n", - "Non-trainable params: 0\n", - "_________________________________________________________________\n" - ] - } - ], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that in the dense layer we only have $1725$ parameters instead of $784 * 625 = 490000$." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "optimizer = optimizers.Adam(lr=1e-2)\n", - "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])" - ] + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "tensor_nets.ipynb", + "version": "0.3.2", + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Train on 60000 samples, validate on 10000 samples\n", - "Epoch 1/2\n", - "60000/60000 [==============================] - 30s 494us/step - loss: 0.2540 - acc: 0.9233 - val_loss: 0.1394 - val_acc: 0.9569\n", - "Epoch 2/2\n", - "60000/60000 [==============================] - 28s 474us/step - loss: 0.1483 - acc: 0.9564 - val_loss: 0.1182 - val_acc: 0.9645\n" - ] - }, - { - "data": { - "text/plain": [ - "" + "cells": [ + { + "metadata": { + "id": "4PbT4UV21L9p", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Tensor Nets (compressing neural networks)\n", + "\n", + "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", + "\n", + "Also, if you run this in Colab, consider switching to GPU: **Runtime -> Change runtime type -> Hardware accelerator -> GPU**.\n", + "\n", + "In this notebook we provide an example of how to build a simple Tensor Net (see https://arxiv.org/abs/1509.06569).\n", + "\n", + "The main ingredient is the so-called TT-Matrix, a generalization of the Kronecker product matrices, i.e. matrices of the form \n", + "$$A = A_1 \\otimes A_2 \\cdots \\otimes A_n$$\n", + "\n", + "In `t3f` TT-Matrices are represented using the `TensorTrain` class." ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.fit(x_train, y_train, epochs=2, batch_size=64, validation_data=(x_test, y_test))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "Compression of Dense layers\n", - "------------------------------------------" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "Let us now train an ordinary DNN (without TT-Matrices) and show how we can compress it using the TT decomposition. (In contrast to directly training a TT-layer from scratch in the example above.)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "model = Sequential()\n", - "model.add(Flatten(input_shape=(28, 28)))\n", - "model.add(Dense(625, activation='relu'))\n", - "model.add(Dense(10))\n", - "model.add(Activation('softmax'))" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "_________________________________________________________________\n", - "Layer (type) Output Shape Param # \n", - "=================================================================\n", - "flatten_5 (Flatten) (None, 784) 0 \n", - "_________________________________________________________________\n", - "dense_6 (Dense) (None, 625) 490625 \n", - "_________________________________________________________________\n", - "dense_7 (Dense) (None, 10) 6260 \n", - "_________________________________________________________________\n", - "activation_8 (Activation) (None, 10) 0 \n", - "=================================================================\n", - "Total params: 496,885\n", - "Trainable params: 496,885\n", - "Non-trainable params: 0\n", - "_________________________________________________________________\n" - ] - } - ], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "optimizer = optimizers.Adam(lr=1e-3)\n", - "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Train on 60000 samples, validate on 10000 samples\n", - "Epoch 1/5\n", - "60000/60000 [==============================] - 7s 111us/step - loss: 0.2937 - acc: 0.9115 - val_loss: 0.1642 - val_acc: 0.9488\n", - "Epoch 2/5\n", - "60000/60000 [==============================] - 6s 102us/step - loss: 0.1353 - acc: 0.9590 - val_loss: 0.1297 - val_acc: 0.9606\n", - "Epoch 3/5\n", - "60000/60000 [==============================] - 6s 97us/step - loss: 0.1027 - acc: 0.9684 - val_loss: 0.0962 - val_acc: 0.9706\n", - "Epoch 4/5\n", - "60000/60000 [==============================] - 6s 100us/step - loss: 0.0813 - acc: 0.9744 - val_loss: 0.1335 - val_acc: 0.9593\n", - "Epoch 5/5\n", - "60000/60000 [==============================] - 6s 100us/step - loss: 0.0736 - acc: 0.9767 - val_loss: 0.1193 - val_acc: 0.9608\n" - ] - }, - { - "data": { - "text/plain": [ - "" + }, + { + "metadata": { + "id": "0Zf7mDAV1L9s", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "eabb2a5e-f0a2-4566-c034-6dac42e19638" + }, + "cell_type": "code", + "source": [ + "import numpy as np\n", + "import tensorflow as tf\n", + "import keras.backend as K\n", + "\n", + "tf.set_random_seed(0)\n", + "np.random.seed(0)\n", + "sess = tf.InteractiveSession()\n", + "K.set_session(sess)\n", + "\n", + "try:\n", + " import t3f\n", + "except ImportError:\n", + " # Install T3F if it's not already installed.\n", + " !git clone https://github.com/Bihaqo/t3f.git\n", + " !cd t3f; pip install .\n", + " import t3f" + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ], + "name": "stderr" + } ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.fit(x_train, y_train, epochs=5, batch_size=64, validation_data=(x_test, y_test))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let us convert the matrix used in the Dense layer to the TT-Matrix with tt-ranks equal to 16 (since we trained the network without the low-rank structure assumption we may wish start with high rank values)." - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "A TT-Matrix of size 784 x 625, underlying tensor shape: (7, 4, 7, 4) x (5, 5, 5, 5), TT-ranks: (1, 16, 16, 16, 1)\n" - ] - } - ], - "source": [ - "W = model.trainable_weights[0]\n", - "print(W)\n", - "Wtt = t3f.to_tt_matrix(W, shape=[[7, 4, 7, 4], [5, 5, 5, 5]], max_tt_rank=16)\n", - "print(Wtt)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We need to evaluate the tt-cores of Wtt. We also need to store other parameters for later (biases and the second dense layer)." - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "cores = sess.run(Wtt.tt_cores)\n", - "other_params = model.get_weights()[1:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we can construct a tensor network with the first Dense layer replaced by `Wtt`\n", - "initialized using the previously computed cores." - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "model = Sequential()\n", - "model.add(Flatten(input_shape=(28, 28)))\n", - "tt_layer = t3f.nn.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5],\n", - " tt_rank=16, activation='relu')\n", - "model.add(tt_layer)\n", - "model.add(Dense(10))\n", - "model.add(Activation('softmax'))" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "optimizer = optimizers.Adam(lr=1e-3)\n", - "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "model.set_weights(list(cores) + other_params)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10000/10000 [==============================] - 10s 972us/step\n", - "new accuracy: 0.6459\n" - ] - } - ], - "source": [ - "print(\"new accuracy: \", model.evaluate(x_test, y_test)[1])" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "_________________________________________________________________\n", - "Layer (type) Output Shape Param # \n", - "=================================================================\n", - "flatten_6 (Flatten) (None, 784) 0 \n", - "_________________________________________________________________\n", - "tt_dense_4 (TTDense) (None, 625) 15585 \n", - "_________________________________________________________________\n", - "dense_8 (Dense) (None, 10) 6260 \n", - "_________________________________________________________________\n", - "activation_10 (Activation) (None, 10) 0 \n", - "=================================================================\n", - "Total params: 21,845\n", - "Trainable params: 21,845\n", - "Non-trainable params: 0\n", - "_________________________________________________________________\n" - ] - } - ], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We see that even though we now have about 5% of the original number of parameters we still achieve a relatively high accuracy." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finetuning the model \n", - "-------------------------------\n", - "We can now finetune this tensor network." - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Train on 60000 samples, validate on 10000 samples\n", - "Epoch 1/2\n", - "60000/60000 [==============================] - 117s 2ms/step - loss: 0.1346 - acc: 0.9595 - val_loss: 0.1142 - val_acc: 0.9653\n", - "Epoch 2/2\n", - "60000/60000 [==============================] - 128s 2ms/step - loss: 0.0806 - acc: 0.9759 - val_loss: 0.0823 - val_acc: 0.9737\n" - ] - }, - { - "data": { - "text/plain": [ - "" + }, + { + "metadata": { + "id": "DTArGyPc1L9w", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "d3246e1b-a40f-4450-e884-b8e02a02da2d" + }, + "cell_type": "code", + "source": [ + "W = t3f.random_matrix([[4, 7, 4, 7], [5, 5, 5, 5]], tt_rank=2)\n", + "\n", + "print(W)" + ], + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "text": [ + "A TT-Matrix of size 784 x 625, underlying tensor shape: (4, 7, 4, 7) x (5, 5, 5, 5), TT-ranks: (1, 2, 2, 2, 1)\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "oQOaaA4R1L91", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Using TT-Matrices we can compactly represent densely connected layers in neural networks, which allows us to greatly reduce number of parameters. Matrix multiplication can be handled by the `t3f.matmul` method which allows for multiplying dense (ordinary) matrices and TT-Matrices. Very simple neural network could look as following (for initialization several options such as `t3f.glorot_initializer`, `t3f.he_initializer` or `t3f.random_matrix` are available):" + ] + }, + { + "metadata": { + "id": "tLvV49Cd1L93", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "x = tf.placeholder(tf.float32, [None, 784])\n", + "y = tf.placeholder(tf.int64, [None])\n", + "\n", + "initializer = t3f.glorot_initializer([[4, 7, 4, 7], [5, 5, 5, 5]], tt_rank=2)\n", + "W1 = t3f.get_variable('W1', initializer=initializer) \n", + "b1 = tf.get_variable('b1', shape=[625])\n", + "h1 = t3f.matmul(x, W1) + b1\n", + "h1 = tf.nn.relu(h1)\n", + "\n", + "W2 = tf.get_variable('W2', shape=[625, 10])\n", + "b2 = tf.get_variable('b2', shape=[10])\n", + "h2 = tf.matmul(h1, W2) + b2\n", + "\n", + "y_ = tf.one_hot(y, 10)\n", + "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=h2))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "XZCtiSnC1L94", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "For convenience we have implemented a layer analogous to *Keras* `Dense` layer but with a TT-Matrix instead of an ordinary matrix. An example of fully trainable net is provided below." ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" + }, + { + "metadata": { + "id": "vJ9KEc201L95", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from keras.datasets import mnist\n", + "from keras.models import Sequential\n", + "from keras.layers import Dense, Activation, Dropout, Flatten\n", + "import numpy as np\n", + "from keras.utils import to_categorical\n", + "from keras import optimizers" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "AVLgpYF_1L99", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + }, + "outputId": "ad2035ad-9dd2-4c8c-a994-bb60433848b4" + }, + "cell_type": "code", + "source": [ + "(x_train, y_train), (x_test, y_test) = mnist.load_data()" + ], + "execution_count": 5, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n", + "11493376/11490434 [==============================] - 3s 0us/step\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "astcob7O1L9-", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Some preprocessing..." + ] + }, + { + "metadata": { + "id": "qoBbdpGP1L9_", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "x_train = x_train / 127.5 - 1.0\n", + "x_test = x_test / 127.5 - 1.0\n", + "\n", + "y_train = to_categorical(y_train, num_classes=10)\n", + "y_test = to_categorical(y_test, num_classes=10)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "CEa9GBp81L-C", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "model = Sequential()\n", + "model.add(Flatten(input_shape=(28, 28)))\n", + "tt_layer = t3f.nn.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5],\n", + " tt_rank=4, activation='relu',\n", + " bias_initializer=1e-3)\n", + "model.add(tt_layer)\n", + "model.add(Dense(10))\n", + "model.add(Activation('softmax'))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "MApNz_T31L-G", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 272 + }, + "outputId": "52d6df65-b4df-4882-f084-eda96c944249" + }, + "cell_type": "code", + "source": [ + "model.summary()" + ], + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "text": [ + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "flatten_1 (Flatten) (None, 784) 0 \n", + "_________________________________________________________________\n", + "keras_dense_1 (KerasDense) (None, 625) 1725 \n", + "_________________________________________________________________\n", + "dense_1 (Dense) (None, 10) 6260 \n", + "_________________________________________________________________\n", + "activation_2 (Activation) (None, 10) 0 \n", + "=================================================================\n", + "Total params: 7,985\n", + "Trainable params: 7,985\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "y4zrSP531L-K", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Note that in the dense layer we only have $1725$ parameters instead of $784 * 625 = 490000$." + ] + }, + { + "metadata": { + "id": "FyqaT09O1L-K", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "optimizer = optimizers.Adam(lr=1e-2)\n", + "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "fF9NrphG1L-O", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 119 + }, + "outputId": "01796db2-22ed-479c-83d0-b2fae278f036" + }, + "cell_type": "code", + "source": [ + "model.fit(x_train, y_train, epochs=2, batch_size=64, validation_data=(x_test, y_test))" + ], + "execution_count": 10, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Train on 60000 samples, validate on 10000 samples\n", + "Epoch 1/2\n", + "60000/60000 [==============================] - 9s 151us/step - loss: 0.2311 - acc: 0.9298 - val_loss: 0.1536 - val_acc: 0.9560\n", + "Epoch 2/2\n", + "60000/60000 [==============================] - 8s 137us/step - loss: 0.1380 - acc: 0.9591 - val_loss: 0.1716 - val_acc: 0.9500\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 10 + } + ] + }, + { + "metadata": { + "collapsed": true, + "id": "J9IVspp61L-R", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Compression of Dense layers\n", + "------------------------------------------" + ] + }, + { + "metadata": { + "collapsed": true, + "id": "mIl7XppU1L-S", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Let us now train an ordinary DNN (without TT-Matrices) and show how we can compress it using the TT decomposition. (In contrast to directly training a TT-layer from scratch in the example above.)" + ] + }, + { + "metadata": { + "id": "FnWQ5MCk1L-S", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "model = Sequential()\n", + "model.add(Flatten(input_shape=(28, 28)))\n", + "model.add(Dense(625, activation='relu'))\n", + "model.add(Dense(10))\n", + "model.add(Activation('softmax'))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "RBS-c6YK1L-V", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 272 + }, + "outputId": "9273bb8a-26f3-419b-c3f8-f747bc7691d9" + }, + "cell_type": "code", + "source": [ + "model.summary()" + ], + "execution_count": 12, + "outputs": [ + { + "output_type": "stream", + "text": [ + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "flatten_2 (Flatten) (None, 784) 0 \n", + "_________________________________________________________________\n", + "dense_2 (Dense) (None, 625) 490625 \n", + "_________________________________________________________________\n", + "dense_3 (Dense) (None, 10) 6260 \n", + "_________________________________________________________________\n", + "activation_3 (Activation) (None, 10) 0 \n", + "=================================================================\n", + "Total params: 496,885\n", + "Trainable params: 496,885\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "SdJIVj5W1L-Y", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "optimizer = optimizers.Adam(lr=1e-3)\n", + "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "Qdv8q1S61L-a", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 221 + }, + "outputId": "16a883ef-439d-4ba4-9856-fbc40e573e03" + }, + "cell_type": "code", + "source": [ + "model.fit(x_train, y_train, epochs=5, batch_size=64, validation_data=(x_test, y_test))" + ], + "execution_count": 14, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Train on 60000 samples, validate on 10000 samples\n", + "Epoch 1/5\n", + "60000/60000 [==============================] - 6s 104us/step - loss: 0.2771 - acc: 0.9156 - val_loss: 0.1529 - val_acc: 0.9528\n", + "Epoch 2/5\n", + "60000/60000 [==============================] - 6s 101us/step - loss: 0.1278 - acc: 0.9613 - val_loss: 0.1079 - val_acc: 0.9680\n", + "Epoch 3/5\n", + "60000/60000 [==============================] - 6s 101us/step - loss: 0.0960 - acc: 0.9702 - val_loss: 0.1078 - val_acc: 0.9658\n", + "Epoch 4/5\n", + "60000/60000 [==============================] - 6s 102us/step - loss: 0.0806 - acc: 0.9744 - val_loss: 0.0948 - val_acc: 0.9714\n", + "Epoch 5/5\n", + "60000/60000 [==============================] - 6s 102us/step - loss: 0.0733 - acc: 0.9770 - val_loss: 0.1072 - val_acc: 0.9664\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 14 + } + ] + }, + { + "metadata": { + "id": "TdLn-FIK1L-g", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Let us convert the matrix used in the Dense layer to the TT-Matrix with tt-ranks equal to 16 (since we trained the network without the low-rank structure assumption we may wish start with high rank values)." + ] + }, + { + "metadata": { + "id": "TBqfSwZf1L-g", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + }, + "outputId": "0a104a46-b6fe-4657-d1cc-2b2c2a55d900" + }, + "cell_type": "code", + "source": [ + "W = model.trainable_weights[0]\n", + "print(W)\n", + "Wtt = t3f.to_tt_matrix(W, shape=[[7, 4, 7, 4], [5, 5, 5, 5]], max_tt_rank=16)\n", + "print(Wtt)" + ], + "execution_count": 15, + "outputs": [ + { + "output_type": "stream", + "text": [ + "\n", + "A TT-Matrix of size 784 x 625, underlying tensor shape: (7, 4, 7, 4) x (5, 5, 5, 5), TT-ranks: (1, 16, 16, 16, 1)\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "IIbCDkyA1L-i", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "We need to evaluate the tt-cores of Wtt. We also need to store other parameters for later (biases and the second dense layer)." + ] + }, + { + "metadata": { + "id": "tscSHv7X1L-i", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "cores = sess.run(Wtt.tt_cores)\n", + "other_params = model.get_weights()[1:]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "llryIpkW1L-l", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Now we can construct a tensor network with the first Dense layer replaced by `Wtt`\n", + "initialized using the previously computed cores." + ] + }, + { + "metadata": { + "id": "i8zQfyNu1L-l", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "model = Sequential()\n", + "model.add(Flatten(input_shape=(28, 28)))\n", + "tt_layer = t3f.nn.KerasDense(input_dims=[7, 4, 7, 4], output_dims=[5, 5, 5, 5],\n", + " tt_rank=16, activation='relu')\n", + "model.add(tt_layer)\n", + "model.add(Dense(10))\n", + "model.add(Activation('softmax'))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "L_BGc2MV1L-p", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "optimizer = optimizers.Adam(lr=1e-3)\n", + "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "BllmgHlt1L-q", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "model.set_weights(list(cores) + other_params)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "NwB7UfTV1L-x", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + }, + "outputId": "d51522c4-441e-4e79-9fb4-ff7970c29eda" + }, + "cell_type": "code", + "source": [ + "print(\"new accuracy: \", model.evaluate(x_test, y_test)[1])" + ], + "execution_count": 20, + "outputs": [ + { + "output_type": "stream", + "text": [ + "10000/10000 [==============================] - 1s 102us/step\n", + "new accuracy: 0.6533\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "Rnnru3s51L-0", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 272 + }, + "outputId": "48ff03a8-dba2-45b2-b9fc-1e0cb0eb9066" + }, + "cell_type": "code", + "source": [ + "model.summary()" + ], + "execution_count": 21, + "outputs": [ + { + "output_type": "stream", + "text": [ + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "flatten_3 (Flatten) (None, 784) 0 \n", + "_________________________________________________________________\n", + "keras_dense_2 (KerasDense) (None, 625) 15585 \n", + "_________________________________________________________________\n", + "dense_4 (Dense) (None, 10) 6260 \n", + "_________________________________________________________________\n", + "activation_5 (Activation) (None, 10) 0 \n", + "=================================================================\n", + "Total params: 21,845\n", + "Trainable params: 21,845\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "Cba_5ExM1L-5", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "We see that even though we now have about 5% of the original number of parameters we still achieve a relatively high accuracy." + ] + }, + { + "metadata": { + "id": "VFCEZjP11L-5", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Finetuning the model \n", + "-------------------------------\n", + "We can now finetune this tensor network." + ] + }, + { + "metadata": { + "id": "268p2zec1L-6", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 119 + }, + "outputId": "ae31a89e-1c86-426e-eb06-a470a6ea2357" + }, + "cell_type": "code", + "source": [ + "model.fit(x_train, y_train, epochs=2, batch_size=64, validation_data=(x_test, y_test))" + ], + "execution_count": 22, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Train on 60000 samples, validate on 10000 samples\n", + "Epoch 1/2\n", + "60000/60000 [==============================] - 12s 196us/step - loss: 0.1353 - acc: 0.9589 - val_loss: 0.0983 - val_acc: 0.9710\n", + "Epoch 2/2\n", + "60000/60000 [==============================] - 11s 177us/step - loss: 0.0810 - acc: 0.9749 - val_loss: 0.0820 - val_acc: 0.9751\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 22 + } + ] + }, + { + "metadata": { + "id": "DX2XYk4J1L_A", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "We see that we were able to achieve higher validation accuracy than we had in the plain DNN, while keeping the number of parameters extremely small (21845 vs 496885 parameters in the uncompressed model)." + ] + }, + { + "metadata": { + "id": "w81lLRov1L_A", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] } - ], - "source": [ - "model.fit(x_train, y_train, epochs=2, batch_size=64, validation_data=(x_test, y_test))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We see that we were able to achieve higher validation accuracy than we had in the plain DNN, while keeping the number of parameters extremely small (21845 vs 496885 parameters in the uncompressed model)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + ] +} \ No newline at end of file From 75ac74b7f443c01d5d0b55328045816dc5c3623d Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 18:46:23 +0000 Subject: [PATCH 185/233] remove unnesassery gpu instructions --- docs/tutorials/tensor_nets.ipynb | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index 23cdcde8..929f60b8 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -25,8 +25,6 @@ "\n", "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", "\n", - "Also, if you run this in Colab, consider switching to GPU: **Runtime -> Change runtime type -> Hardware accelerator -> GPU**.\n", - "\n", "In this notebook we provide an example of how to build a simple Tensor Net (see https://arxiv.org/abs/1509.06569).\n", "\n", "The main ingredient is the so-called TT-Matrix, a generalization of the Kronecker product matrices, i.e. matrices of the form \n", From 8abdc5144aaf2513c78772fa3c107ea41c183dbd Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 18:48:20 +0000 Subject: [PATCH 186/233] add nn module to API doc --- docs/api.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index 19cea1ac..4a92efa5 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -17,6 +17,15 @@ Module contents :show-inheritance: +t3f\.nn module +----------------- + +.. automodule:: t3f.nn + :members: + :undoc-members: + :show-inheritance: + + t3f\.utils module ----------------- From 28ff03c2db75d313186491d15e789f8ee1f97303 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 18:53:43 +0000 Subject: [PATCH 187/233] note on Keras dependency --- docs/installation.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 8c02d608..be60b216 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -21,4 +21,7 @@ To install the latest version, run git clone https://github.com/Bihaqo/t3f.git cd t3f - pip install . \ No newline at end of file + pip install . + +Note that using the neural module (`t3f.nn`) may require additional dependencies +such as Keras. From 7a64a8cfb88b0b22ce502f56639c08c1435f25b5 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 18:53:58 +0000 Subject: [PATCH 188/233] identation fix --- t3f/nn.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/t3f/nn.py b/t3f/nn.py index c59bdb31..1d8e3ebb 100644 --- a/t3f/nn.py +++ b/t3f/nn.py @@ -17,19 +17,19 @@ def __init__(self, input_dims, output_dims, tt_rank=2, """Creates a TT-Matrix based Dense Keras layer. Args: - input_dims: an array, tensor shape of the matrix row index - ouput_dims: an array, tensor shape of the matrix column index - tt_rank: a number or an array, desired tt-rank of the TT-Matrix - activation: [None] string or None, specifies the activation function. - use_bias: bool, whether to use bias - kernel_initializer: string specifying initializer for the TT-Matrix. - Possible values are 'glorot', 'he', and 'lecun'. - bias_initializer: a number, initialization value of the bias + input_dims: an array, tensor shape of the matrix row index + ouput_dims: an array, tensor shape of the matrix column index + tt_rank: a number or an array, desired tt-rank of the TT-Matrix + activation: [None] string or None, specifies the activation function. + use_bias: bool, whether to use bias + kernel_initializer: string specifying initializer for the TT-Matrix. + Possible values are 'glorot', 'he', and 'lecun'. + bias_initializer: a number, initialization value of the bias Returns: - Layer object corresponding to multiplication by a TT-Matrix - followed by addition of a bias and applying - an elementwise activation + Layer object corresponding to multiplication by a TT-Matrix + followed by addition of a bias and applying + an elementwise activation Raises: ValueError if the provided activation or kernel_initializer is From ee3958f49b1ab90b9b3a995f035ebcb30b9dde36 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 19:09:04 +0000 Subject: [PATCH 189/233] some more notes on other libraries --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b7a1fac1..110f87c8 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,10 @@ TensorFlow implementation of the Tensor Train (TT) -Toolbox. The documentation is available via [readthedocs](https://t3f.readthedocs.io/en/latest/index.html). # Comparison with other libraries -TODO: list of libraries, pros and cons, benchmarking? Or maybe just link to the documentation? -There are also implementations of the TT-toolbox in [plain Python](https://github.com/oseledets/ttpy) and [Matlab](https://github.com/oseledets/TT-Toolbox). +There are implementations of the TT-toolbox in [plain Python](https://github.com/oseledets/ttpy) and [Matlab](https://github.com/oseledets/TT-Toolbox). Also, there is a very nice generic tensor network library [tnttorch](https://github.com/rballester/tntorch) which supports TT as a special case. + +The main difference between `t3f` Python/Matlab implementations is that `t3f` uses TensorFlow as backend and thus supports GPUs, automatic differentiation, and batch processing. As a more generic library, `tnttorch` lacks some TensorTrain specific tools (e.g. Riemannian optimization support). + Here are the results im ms of benchmarking T3F on CPU and GPU and comparing against the [TTPY library](https://github.com/oseledets/ttpy) From b5b09b43608a8f50505ab67950fdadc486a2a4d1 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 29 Dec 2018 21:08:43 +0000 Subject: [PATCH 190/233] init better comparison to other libraries --- docs/comparison.rst | 35 +++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 36 insertions(+) create mode 100644 docs/comparison.rst diff --git a/docs/comparison.rst b/docs/comparison.rst new file mode 100644 index 00000000..e7b31729 --- /dev/null +++ b/docs/comparison.rst @@ -0,0 +1,35 @@ +Comparison to other libraries +============================= + +A brief overview of other libraries that support Tensor Train decomposition (which is also known under the name Matrix Product State in physics community). + ++---------------+-------------------+---------+------------+--------------+-----------+ +| | Library | | Language | | GPU | | autodiff | | Riemannian | | DMRG | +| | | | | | | | | | | | AMen | +| | | | | | | | | | | | TT-cross| ++===============+===================+=========+============+==============+===========+ +| t3f | Python/TensorFlow | Yes | Yes | Yes | No | ++---------------+-------------------+---------+------------+--------------+-----------+ +| tntorch_ | Python/PyTorch | Yes | Yes | No | No | ++---------------+-------------------+---------+------------+--------------+-----------+ +| ttpy_ | Python | No | No | Yes | Yes | ++---------------+-------------------+---------+------------+--------------+-----------+ +| mpnum_ | Python | No | No | No | DMRG | ++---------------+-------------------+---------+------------+--------------+-----------+ +| `scikit_tt`_ | Python | No | No | No | No | ++---------------+-------------------+---------+------------+--------------+-----------+ +| mpys_ | Python | No | No | No | No | ++---------------+-------------------+---------+------------+--------------+-----------+ +| `TT-Toolbox`_ | Matlab | Partial | No | No | Yes | ++---------------+-------------------+---------+------------+--------------+-----------+ +| ITensor_ | C++ | No | No | No | DMRG | ++---------------+-------------------+---------+------------+--------------+-----------+ + + +.. _tntorch: https://github.com/rballester/tntorch +.. _ttpy: https://github.com/oseledets/ttpy +.. _mpnum: https://github.com/dseuss/mpnum +.. _scikit\_tt: https://github.com/PGelss/scikit_tt +.. _mpys: https://github.com/alvarorga/mpys +.. _TT-Toolbox: https://github.com/oseledets/TT-Toolbox +.. _ITensor: http://itensor.org/ \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index d32664bc..52a26235 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ t3f is implemented on top of TensorFlow which gives it a few nice properties: quick_start faq api + comparison .. toctree:: :maxdepth: 1 From 412e6a1ad1f6dc14ca38503399b1d56abdf8b8bb Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sat, 5 Jan 2019 00:06:59 +0000 Subject: [PATCH 191/233] Which library to choose and benchmarking notes --- docs/comparison.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/comparison.rst b/docs/comparison.rst index e7b31729..7f08e146 100644 --- a/docs/comparison.rst +++ b/docs/comparison.rst @@ -32,4 +32,8 @@ A brief overview of other libraries that support Tensor Train decomposition (whi .. _scikit\_tt: https://github.com/PGelss/scikit_tt .. _mpys: https://github.com/alvarorga/mpys .. _TT-Toolbox: https://github.com/oseledets/TT-Toolbox -.. _ITensor: http://itensor.org/ \ No newline at end of file +.. _ITensor: http://itensor.org/ + +If you use python, we would suggest using t3f if you need extensive Riemannian optimization support, t3f or tntorch if you need GPU or autodiff support, and ttpy if you need advanced algorithms such as AMen. + +The performance of the libraries is a bit tricky to measure fairly and is actually not that different between the libraries because everyone relies on the same BLAS/MKL subruitines. However, GPU can help a lot if you need operations that can be expressed as large matrix-by-matrix multiplications, e.g. computing a gram matrix of a bunch of tensors. For more details on benchmarking t3f see :doc:`benchmarking`. From ccd9d1963c74a685b00afba003db72ac78e88f30 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 14:15:40 +0000 Subject: [PATCH 192/233] 3 more libraries to comparison --- docs/comparison.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/comparison.rst b/docs/comparison.rst index 7f08e146..4183fb35 100644 --- a/docs/comparison.rst +++ b/docs/comparison.rst @@ -22,8 +22,14 @@ A brief overview of other libraries that support Tensor Train decomposition (whi +---------------+-------------------+---------+------------+--------------+-----------+ | `TT-Toolbox`_ | Matlab | Partial | No | No | Yes | +---------------+-------------------+---------+------------+--------------+-----------+ +| TENSORBOX_ | Matlab | Partial | No | ?? | ?? | ++---------------+-------------------+---------+------------+--------------+-----------+ +| Tensorlab_ | Matlab | Partial | No | ?? | ?? | ++---------------+-------------------+---------+------------+--------------+-----------+ | ITensor_ | C++ | No | No | No | DMRG | +---------------+-------------------+---------+------------+--------------+-----------+ +| libtt_ | C++ | No | No | No | TT-cross | ++---------------+-------------------+---------+------------+--------------+-----------+ .. _tntorch: https://github.com/rballester/tntorch @@ -32,7 +38,10 @@ A brief overview of other libraries that support Tensor Train decomposition (whi .. _scikit\_tt: https://github.com/PGelss/scikit_tt .. _mpys: https://github.com/alvarorga/mpys .. _TT-Toolbox: https://github.com/oseledets/TT-Toolbox +.. _TENSORBOX: http://www.bsp.brain.riken.jp/~phan/#tensorbox +.. _Tensorlab: https://www.tensorlab.net .. _ITensor: http://itensor.org/ +.. _libtt: https://bitbucket.org/matseralex/tt_smoluh/src/master/libtt/ If you use python, we would suggest using t3f if you need extensive Riemannian optimization support, t3f or tntorch if you need GPU or autodiff support, and ttpy if you need advanced algorithms such as AMen. From d7851bf05cef102b93409f6516e72795040b2502 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 14:16:06 +0000 Subject: [PATCH 193/233] Note that TT and MPS are the same --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 110f87c8..d5a63158 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [![Build Status](https://travis-ci.org/Bihaqo/t3f.svg?branch=develop)](https://travis-ci.org/Bihaqo/t3f) [![Coverage Status](https://coveralls.io/repos/github/Bihaqo/t3f/badge.svg?branch=develop)](https://coveralls.io/github/Bihaqo/t3f?branch=develop) -TensorFlow implementation of the Tensor Train (TT) -Toolbox. +TensorFlow implementation of a library for working with Tensor Train (TT) decomposition which is also known as Matrix Product State (MPS). # Documentation The documentation is available via [readthedocs](https://t3f.readthedocs.io/en/latest/index.html). From 1e739df5c266a5d195ae80ceb8ab658a477457a1 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 14:59:15 +0000 Subject: [PATCH 194/233] regression test that runtime_check = False also works --- t3f/autodiff_test.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index 95eda504..67dcc780 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -9,6 +9,15 @@ class _AutodiffTest(): + def _check_single_gradient(self, func, x, desired): + actual1 = ops.full(autodiff.gradients(func, x, runtime_check=False)) + actual2 = ops.full(autodiff.gradients(func, x, runtime_check=True)) + + with self.test_session() as sess: + desired_v, actual1_v, actual2_v = sess.run([desired, actual1, actual2]) + self.assertAllClose(desired_v, actual1_v, rtol=1e-4) + self.assertAllClose(desired_v, actual2_v, rtol=1e-4) + def testGradients(self): w = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) A = initializers.random_matrix(([5] * 3, [5] * 3), dtype=self.dtype) @@ -16,22 +25,14 @@ def testGradients(self): def func1(x): return 0.5 * ops.flat_inner(x, w) ** 2 - - actual1 = ops.full(autodiff.gradients(func1, x)) desired1 = ops.full(ops.flat_inner(x, w) * riemannian.project(w, x)) - with self.test_session() as sess: - desired1_v, actual1_v = sess.run([desired1, actual1]) - np.testing.assert_allclose(desired1_v, actual1_v, rtol=1e-4) + self._check_single_gradient(func1, x, desired1) def func2(x): return ops.quadratic_form(A, x, x) - - actual2 = ops.full(autodiff.gradients(func2, x)) grad = ops.matmul(ops.transpose(A) + A, x) desired2 = ops.full(riemannian.project(grad, x)) - with self.test_session() as sess: - desired2_v, actual2_v = sess.run([desired2, actual2]) - np.testing.assert_allclose(desired2_v, actual2_v, rtol=1e-4) + self._check_single_gradient(func2, x, desired2) def func3(x): # A function which is not invariant to different representations of the @@ -42,6 +43,16 @@ def func3(x): with self.test_session() as sess: sess.run(actual3) + def _check_single_hessian_by_vector(self, func, x, z, desired): + actual1 = ops.full(autodiff.hessian_vector_product( + func, x, z, runtime_check=False)) + actual2 = ops.full(autodiff.hessian_vector_product(func, x, z, + runtime_check=True)) + + with self.test_session() as sess: + desired_v, actual1_v, actual2_v = sess.run([desired, actual1, actual2]) + self.assertAllClose(desired_v, actual1_v, rtol=1e-4) + self.assertAllClose(desired_v, actual2_v, rtol=1e-4) def testHessianVectorProduct(self): w = initializers.random_matrix(([5] * 3, None), dtype=self.dtype) @@ -55,23 +66,16 @@ def func1(x): # Grad: w # Hessian: w w.T # Hessian by vector: w - - actual1 = ops.full(autodiff.hessian_vector_product(func1, x, z)) desired1 = riemannian.project(ops.flat_inner(projected_vector, w) * w, x) desired1 = ops.full(desired1) - with self.test_session() as sess: - desired1_v, actual1_v = sess.run([desired1, actual1]) - np.testing.assert_allclose(desired1_v, actual1_v, rtol=1e-4) + self._check_single_hessian_by_vector(func1, x, z, desired1) def func2(x): return ops.quadratic_form(A, x, x) # Hessian of is A + A.T - actual2 = ops.full(autodiff.hessian_vector_product(func2, x, z)) hessian_by_vector = ops.matmul(ops.transpose(A) + A, projected_vector) desired2 = ops.full(riemannian.project(hessian_by_vector, x)) - with self.test_session() as sess: - desired2_v, actual2_v = sess.run([desired2, actual2]) - np.testing.assert_allclose(desired2_v, actual2_v, rtol=1e-4) + self._check_single_hessian_by_vector(func1, x, z, desired1) def func3(x): # A function which is not invariant to different representations of the From 11801169037a57c108edbdd93abd56b96f15f6d6 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 15:03:06 +0000 Subject: [PATCH 195/233] better method naming --- t3f/autodiff_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index 67dcc780..257cef8f 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -9,7 +9,7 @@ class _AutodiffTest(): - def _check_single_gradient(self, func, x, desired): + def _TestSingleGradient(self, func, x, desired): actual1 = ops.full(autodiff.gradients(func, x, runtime_check=False)) actual2 = ops.full(autodiff.gradients(func, x, runtime_check=True)) @@ -26,13 +26,13 @@ def testGradients(self): def func1(x): return 0.5 * ops.flat_inner(x, w) ** 2 desired1 = ops.full(ops.flat_inner(x, w) * riemannian.project(w, x)) - self._check_single_gradient(func1, x, desired1) + self._TestSingleGradient(func1, x, desired1) def func2(x): return ops.quadratic_form(A, x, x) grad = ops.matmul(ops.transpose(A) + A, x) desired2 = ops.full(riemannian.project(grad, x)) - self._check_single_gradient(func2, x, desired2) + self._TestSingleGradient(func2, x, desired2) def func3(x): # A function which is not invariant to different representations of the @@ -43,7 +43,7 @@ def func3(x): with self.test_session() as sess: sess.run(actual3) - def _check_single_hessian_by_vector(self, func, x, z, desired): + def _TestSingleHessianByVector(self, func, x, z, desired): actual1 = ops.full(autodiff.hessian_vector_product( func, x, z, runtime_check=False)) actual2 = ops.full(autodiff.hessian_vector_product(func, x, z, @@ -68,14 +68,14 @@ def func1(x): # Hessian by vector: w desired1 = riemannian.project(ops.flat_inner(projected_vector, w) * w, x) desired1 = ops.full(desired1) - self._check_single_hessian_by_vector(func1, x, z, desired1) + self._TestSingleHessianByVector(func1, x, z, desired1) def func2(x): return ops.quadratic_form(A, x, x) # Hessian of is A + A.T hessian_by_vector = ops.matmul(ops.transpose(A) + A, projected_vector) desired2 = ops.full(riemannian.project(hessian_by_vector, x)) - self._check_single_hessian_by_vector(func1, x, z, desired1) + self._TestSingleHessianByVector(func1, x, z, desired1) def func3(x): # A function which is not invariant to different representations of the From 0c63ef38d15927f83a82b8d3651625133fe0b18e Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 15:28:11 +0000 Subject: [PATCH 196/233] move benchmark_config implementation into separate file --- examples/profile/profile.py | 21 ++------------ examples/profile/tmp_benchmark_config.py | 37 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 19 deletions(-) create mode 100644 examples/profile/tmp_benchmark_config.py diff --git a/examples/profile/profile.py b/examples/profile/profile.py index e9145af5..75aa338c 100644 --- a/examples/profile/profile.py +++ b/examples/profile/profile.py @@ -3,25 +3,7 @@ import pickle import argparse import tensorflow as tf - - -# TODO: remove this after the next release of TF (which should include -# tf.test.benchmark_config()) -try: - tf.test.benchmark_config() -except AttributeError: - from tensorflow import core - def benchmark_config(): - """Returns a tf.ConfigProto for disabling the dependency optimizer. - Returns: - A TensorFlow ConfigProto object. - """ - config = core.protobuf.config_pb2.ConfigProto() - config.graph_options.rewrite_options.dependency_optimization = ( - core.protobuf.rewriter_config_pb2.RewriterConfig.OFF) - return config - tf.test.benchmark_config = benchmark_config - +import tmp_benchmark_config from tensorflow.python.client import device_lib import t3f @@ -45,6 +27,7 @@ def benchmark_config(): vecs100 = t3f.cast(vecs100, tf.float64) one_vec100 = t3f.get_variable('one_vec100', initializer=vecs100[0]) vecs100 = t3f.get_variable('vecs100', initializer=vecs100) +tmp_benchmark_config.import_benchmark_config() sess = tf.Session(config=tf.test.benchmark_config()) sess.run(tf.global_variables_initializer()) print(device_lib.list_local_devices()) diff --git a/examples/profile/tmp_benchmark_config.py b/examples/profile/tmp_benchmark_config.py new file mode 100644 index 00000000..7c63c76d --- /dev/null +++ b/examples/profile/tmp_benchmark_config.py @@ -0,0 +1,37 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""A copy of tf.test.benchmark_config() to be used until next stable release. +Copied with minor modifications from +https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/platform/benchmark.py +""" +import tensorflow as tf + + +def import_benchmark_config(): + try: + tf.test.benchmark_config() + except AttributeError: + from tensorflow import core + def benchmark_config(): + """Returns a tf.ConfigProto for disabling the dependency optimizer. + Returns: + A TensorFlow ConfigProto object. + """ + config = core.protobuf.config_pb2.ConfigProto() + config.graph_options.rewrite_options.dependency_optimization = ( + core.protobuf.rewriter_config_pb2.RewriterConfig.OFF) + return config + tf.test.benchmark_config = benchmark_config From 10d3f74430b59ba2cf25dc5d96226d406f54cfa0 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 16:42:32 +0000 Subject: [PATCH 197/233] mv examples/profile -> docs/benchmark --- {examples/profile => docs/benchmark}/README.md | 0 .../profile.py => docs/benchmark/benchmark.py | 0 .../benchmark/benchmark_ttpy.py | 0 {examples/profile => docs/benchmark}/logs_cpu.pkl | Bin {examples/profile => docs/benchmark}/logs_gpu.pkl | Bin {examples/profile => docs/benchmark}/logs_ttpy.pkl | Bin {examples/profile => docs/benchmark}/results.ipynb | 0 {examples/profile => docs/benchmark}/results.png | Bin .../benchmark}/tmp_benchmark_config.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename {examples/profile => docs/benchmark}/README.md (100%) rename examples/profile/profile.py => docs/benchmark/benchmark.py (100%) rename examples/profile/profile_ttpy.py => docs/benchmark/benchmark_ttpy.py (100%) rename {examples/profile => docs/benchmark}/logs_cpu.pkl (100%) rename {examples/profile => docs/benchmark}/logs_gpu.pkl (100%) rename {examples/profile => docs/benchmark}/logs_ttpy.pkl (100%) rename {examples/profile => docs/benchmark}/results.ipynb (100%) rename {examples/profile => docs/benchmark}/results.png (100%) rename {examples/profile => docs/benchmark}/tmp_benchmark_config.py (100%) diff --git a/examples/profile/README.md b/docs/benchmark/README.md similarity index 100% rename from examples/profile/README.md rename to docs/benchmark/README.md diff --git a/examples/profile/profile.py b/docs/benchmark/benchmark.py similarity index 100% rename from examples/profile/profile.py rename to docs/benchmark/benchmark.py diff --git a/examples/profile/profile_ttpy.py b/docs/benchmark/benchmark_ttpy.py similarity index 100% rename from examples/profile/profile_ttpy.py rename to docs/benchmark/benchmark_ttpy.py diff --git a/examples/profile/logs_cpu.pkl b/docs/benchmark/logs_cpu.pkl similarity index 100% rename from examples/profile/logs_cpu.pkl rename to docs/benchmark/logs_cpu.pkl diff --git a/examples/profile/logs_gpu.pkl b/docs/benchmark/logs_gpu.pkl similarity index 100% rename from examples/profile/logs_gpu.pkl rename to docs/benchmark/logs_gpu.pkl diff --git a/examples/profile/logs_ttpy.pkl b/docs/benchmark/logs_ttpy.pkl similarity index 100% rename from examples/profile/logs_ttpy.pkl rename to docs/benchmark/logs_ttpy.pkl diff --git a/examples/profile/results.ipynb b/docs/benchmark/results.ipynb similarity index 100% rename from examples/profile/results.ipynb rename to docs/benchmark/results.ipynb diff --git a/examples/profile/results.png b/docs/benchmark/results.png similarity index 100% rename from examples/profile/results.png rename to docs/benchmark/results.png diff --git a/examples/profile/tmp_benchmark_config.py b/docs/benchmark/tmp_benchmark_config.py similarity index 100% rename from examples/profile/tmp_benchmark_config.py rename to docs/benchmark/tmp_benchmark_config.py From db187f2520fccc8da3d3880f39d4257b023f825a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 16:44:20 +0000 Subject: [PATCH 198/233] remove remainings of examples --- examples/README.md | 12 -- examples/tt-basic.ipynb | 310 ------------------------------------- examples/utils/__init__.py | 1 - 3 files changed, 323 deletions(-) delete mode 100644 examples/README.md delete mode 100644 examples/tt-basic.ipynb delete mode 100644 examples/utils/__init__.py diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 1cf39f70..00000000 --- a/examples/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Examples -This folder contains examples of basic T3f usage such as: - -- Converting a numpy tensor to the Tensor Train format, rounding, various -operations with TT tensors. - -- An example of a Tensor Net - deep neural network based on -TT-Matrices. - -- An example of solving a Tensor Completion problem using tensorflow optimizers - -- An example of solving the Approximation problem using Riemannian optimization diff --git a/examples/tt-basic.ipynb b/examples/tt-basic.ipynb deleted file mode 100644 index 58c6c5c1..00000000 --- a/examples/tt-basic.ipynb +++ /dev/null @@ -1,310 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import t3f\n", - "import tensorflow as tf\n", - "import numpy as np\n", - "\n", - "sess = tf.InteractiveSession()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this example we will show how to convert existing tensors into Tensor Train format. Let us consider the tensor obtained by evaluating $\\sin(x)$ on a uniform grid of size $2 ^ d$ and reshaping the obtained array into tensor of size $2 \\times 2 \\times 2 \\cdots \\times 2$. This is an example of a tensor in the so-called Quantized Tensor Train format (QTT)." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "d = 10\n", - "x = np.linspace(0, 2 * np.pi, 2 ** 10)\n", - "y = np.sin(x)\n", - "y = np.reshape(y, d * [2])" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(2, 2, 2, 2, 2, 2, 2, 2, 2, 2)\n" - ] - } - ], - "source": [ - "print (y.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now lets construct a TT tensor out of `y` using `t3f.to_tt_tensor`." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "A Tensor Train of shape (2, 2, 2, 2, 2, 2, 2, 2, 2, 2), TT-ranks: (1, 2, 4, 8, 16, 32, 16, 8, 4, 2, 1)\n" - ] - } - ], - "source": [ - "y_tt = t3f.to_tt_tensor(y, max_tt_rank=32)\n", - "print (y_tt)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let us verify that this indeed an approximation of the original tensor `y`." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "relative error is 2.8373670049013647e-15\n" - ] - } - ], - "source": [ - "y_tt_full = sess.run(t3f.full(y_tt))\n", - "print (\"relative error is {}\".format(np.linalg.norm(y_tt_full - y) / np.linalg.norm(y)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It is known that the exact TT-rank of the obtained tensor is equal to $2$. Let us round `y_tt` using `t3f.round` by setting the maximal value of the TT-rank to $2$ and verify it numerically." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "relative error is 1.4717721230311198e-15\n" - ] - } - ], - "source": [ - "y_tt_round = t3f.round(y_tt, max_tt_rank=2)\n", - "\n", - "y_tt_round_full = sess.run(t3f.full(y_tt_round))\n", - "print (\"relative error is {}\".format(np.linalg.norm(y_tt_round_full - y) / np.linalg.norm(y)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This fact allows us to significantly compress the tensor `y`. The total number of degrees of freedom in `y_tt` after rounding is :" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "72\n" - ] - } - ], - "source": [ - "dof = sum(np.prod(tt_core.shape.as_list()) for tt_core in y_tt_round.tt_cores)\n", - "print (dof)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "which is much smaller than the original $2^d$ and is in fact proportional to $d$." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Exercise**: perform the same analysis for $e^x$." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "We can also perform various operations with tensors in the TT format." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "# Element-wise product\n", - "y_tt_sq = y_tt_round * y_tt_round\n", - "# Element-wise sum\n", - "y_tt_sum = y_tt_round + y_tt_round" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let us check that we indeed get the desired results" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "y_tt_sq_full = sess.run(t3f.full(y_tt_sq))" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "5.57568393209e-14\n" - ] - } - ], - "source": [ - "print (np.linalg.norm(y ** 2 - y_tt_sq_full))" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "y_tt_sum_full = sess.run(t3f.full(y_tt_sum))" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6.65826459441e-14\n" - ] - } - ], - "source": [ - "print (np.linalg.norm(2 * y - y_tt_sum_full))" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "not enough arguments for format string", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"ololo %a %d\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m: not enough arguments for format string" - ] - } - ], - "source": [ - "print (\"ololo %a %d\" % 1, 3)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Environment (conda_tf)", - "language": "python", - "name": "conda_tf" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/utils/__init__.py b/examples/utils/__init__.py deleted file mode 100644 index dfa90ed7..00000000 --- a/examples/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .tt_dense import TTDense From 3df6b867dba811fe1c663a01260aef018ffe2b10 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 18:29:54 +0000 Subject: [PATCH 199/233] add rst form of the table --- docs/benchmark/results.ipynb | 49 +++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/docs/benchmark/results.ipynb b/docs/benchmark/results.ipynb index 39bd08ff..ad78a9eb 100644 --- a/docs/benchmark/results.ipynb +++ b/docs/benchmark/results.ipynb @@ -2,17 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "import pickle\n", - "import pandas as pd" + "import pandas as pd\n", + "import tabulate" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -88,16 +89,16 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "df = load('logs_cpu.pkl', 'fixed_logs_gpu.pkl', ttpy_path='logs_ttpy.pkl')" + "df = load('logs_cpu.pkl', 'logs_gpu.pkl', ttpy_path='logs_ttpy.pkl')" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -199,7 +200,7 @@ "project_rank100 0.226 " ] }, - "execution_count": 25, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -210,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -237,12 +238,36 @@ "print(df.round(decimals=3).to_latex())" ] }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=============== ================== ============ ============ ============== ==============\n", + ".. ttpy, one on CPU one on CPU one on GPU batch on CPU batch on GPU\n", + "=============== ================== ============ ============ ============== ==============\n", + "matvec 11.142 1.19 0.744 1.885 0.14\n", + "matmul 86.191 9.849 0.95 17.483 1.461\n", + "norm 3.79 2.136 1.019 0.253 0.044\n", + "round 73.027 86.04 165.969 8.234 161.102\n", + "gram 0.145 0.606 0.973 0.021 0.001\n", + "project_rank100 116.868 3.001 13.239 1.645 0.226\n", + "=============== ================== ============ ============ ============== ==============\n" + ] + } + ], + "source": [ + "print(tabulate.tabulate(df.round(decimals=3), headers=df.keys(), tablefmt=\"rst\"))" + ] + }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [] } @@ -263,7 +288,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.5" + "version": "3.6.5" } }, "nbformat": 4, From 9af11b362bc1c8fd1f996eed0872aea6668f6ef8 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 18:30:19 +0000 Subject: [PATCH 200/233] init benchmark doc page --- docs/benchmark.rst | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 docs/benchmark.rst diff --git a/docs/benchmark.rst b/docs/benchmark.rst new file mode 100644 index 00000000..218cb93a --- /dev/null +++ b/docs/benchmark.rst @@ -0,0 +1,33 @@ +Benchmark +================ + +The performance of different libraries implementing Tensor Train decomposition is a bit tricky to compare fairly and is actually not that different because everyone relies on the same BLAS/MKL subruitines. + +So the main purpose of this section is not to prove that T3F is faster than every other library (it is not), but rather to assess GPU gains on different ops and identify bottlenecks by comparing to *some* other library. As a reference implementation, we decided to use ttpy_ library. + +.. _ttpy: https://github.com/oseledets/ttpy + +See the following table for time in ms of different opeartions run in ttpy_ (second column) and in t3f (other columns). + +=============== ================== ============ ============ ============== ============== +Operation ttpy, one on CPU one on CPU one on GPU batch on CPU batch on GPU +=============== ================== ============ ============ ============== ============== +matvec 11.142 1.19 0.744 1.885 0.14 +matmul 86.191 9.849 0.95 17.483 1.461 +norm 3.79 2.136 1.019 0.253 0.044 +round 73.027 86.04 165.969 8.234 161.102 +gram 0.145 0.606 0.973 0.021 0.001 +project_rank100 116.868 3.001 13.239 1.645 0.226 +=============== ================== ============ ============ ============== ============== + +The timing in the "batch" columns represent running the operation for a 100 of objects at the same time and then reporting the time per object. E.g. the last number in the first row (0.14) means that multiplying a single TT-matrix by 100 different TT-vectors takes 14 ms on GPU when using T3F, which translates to 0.14 ms per vector. + +Note that rounding operation is slow on GPU. This is a `known TensorFlow bug`_, that the SVD implementation is slower on GPU than on CPU. + +.. _`known TensorFlow bug`: https://github.com/tensorflow/tensorflow/issues/13603 + +The benchmark was run on NVIDIA DGX-1 server with Tesla V100 GPU and Intel(R) Xeon(R) CPU E5-2698 v4 @ 2.20GHz with 80 logical cores + +To run this benchmark on your own hardware, see `docs/benchmark`_ folder. + +.. _`docs/benchmark`: https://github.com/Bihaqo/t3f/tree/develop/docs/benchmark From 7286b79edb6c988bb4ba5e32582062ae233c4e96 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 18:30:39 +0000 Subject: [PATCH 201/233] remove redundant points that are present in the doc page --- docs/benchmark/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/benchmark/README.md b/docs/benchmark/README.md index 608d35b5..5b0d6388 100644 --- a/docs/benchmark/README.md +++ b/docs/benchmark/README.md @@ -9,10 +9,8 @@ CUDA_VISIBLE_DEVICES=0 python profile.py --file_path logs_gpu.pkl To visualize the results in a table, see ```results.ipynb``` Jupyter notebook. Here are the numbers you can get on NVIDIA DGX-1 server with Tesla V100 GPU and Intel(R) Xeon(R) CPU E5-2698 v4 @ 2.20GHz with 80 logical cores - -The timing is reported in ms and in the batch case it's time per object (e.g. it takes 0.3 ms to perform matrix-by-vector multiplication for a batch of 100 vectors on a CPU, but we report 0.003 = 0.3 / 100 in the table). -Note that by default TensorFlow has a very slow ```tf.transpose``` op for tensors of more than 5 dimensions, which affects the results a lot. To achieve the performance as described above, apply [the following patch](https://github.com/tensorflow/tensorflow/pull/15893) and [compile TensorFlow from sources](https://www.tensorflow.org/install/install_sources). +[More details](https://t3f.readthedocs.io/en/latest/benchmark.html) on how to interpret this numbers see. ## Comparing against TTPY To benchmark T3F against another library for Tensor Train decomposition [TTPY](https://github.com/oseledets/ttpy), install TTPY and run the following command in the bash shell From 7af8a55c58ac62d3e68498ee3051836abe446121 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 18:31:03 +0000 Subject: [PATCH 202/233] add benchmark to the table of contents --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 52a26235..226d4d8d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,7 @@ t3f is implemented on top of TensorFlow which gives it a few nice properties: faq api comparison + benchmark .. toctree:: :maxdepth: 1 From b3a195281d68bc3a27c34269eabf1be565fbf3e2 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 18:32:33 +0000 Subject: [PATCH 203/233] s/profile/benchmark/ --- docs/benchmark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/benchmark/README.md b/docs/benchmark/README.md index 5b0d6388..ca379e28 100644 --- a/docs/benchmark/README.md +++ b/docs/benchmark/README.md @@ -1,5 +1,5 @@ -## Profiling T3F -To profile the library, use the following commands +## Benchmarking T3F +To benchmark the library, use the following commands ```bash # Running on CPU. CUDA_VISIBLE_DEVICES= python profile.py --file_path logs_cpu.pkl From 5f1c918894ab2c956faf23d60f75c5fc8951a0a3 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 18:33:12 +0000 Subject: [PATCH 204/233] prepare for the upcoming list of tested tf versions --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index be60b216..98a73bf4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -3,7 +3,7 @@ Installation ============ -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation of version >= 1.2 (tested versions are from 1.2 to 1.11, see here_ for TF installation instructions). +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation (tested versions are from 1.10 to 1.12, see here_ for TF installation instructions). .. _here: https://www.tensorflow.org/install/ From b7cb0c87da07e03afddefb2fc7ef42d33dbeda29 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 19:03:41 +0000 Subject: [PATCH 205/233] fix link to benchmark --- docs/comparison.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/comparison.rst b/docs/comparison.rst index 4183fb35..6029b9e7 100644 --- a/docs/comparison.rst +++ b/docs/comparison.rst @@ -45,4 +45,4 @@ A brief overview of other libraries that support Tensor Train decomposition (whi If you use python, we would suggest using t3f if you need extensive Riemannian optimization support, t3f or tntorch if you need GPU or autodiff support, and ttpy if you need advanced algorithms such as AMen. -The performance of the libraries is a bit tricky to measure fairly and is actually not that different between the libraries because everyone relies on the same BLAS/MKL subruitines. However, GPU can help a lot if you need operations that can be expressed as large matrix-by-matrix multiplications, e.g. computing a gram matrix of a bunch of tensors. For more details on benchmarking t3f see :doc:`benchmarking`. +The performance of the libraries is a bit tricky to measure fairly and is actually not that different between the libraries because everyone relies on the same BLAS/MKL subruitines. However, GPU can help a lot if you need operations that can be expressed as large matrix-by-matrix multiplications, e.g. computing a gram matrix of a bunch of tensors. For more details on benchmarking t3f see :doc:`benchmark`. From 9c8e68b8fd4796bc1a9cf7e9df7f5fa9f7ada42a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 19:40:40 +0000 Subject: [PATCH 206/233] not ready for this change of versions yet --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 98a73bf4..a2a1ade8 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -3,7 +3,7 @@ Installation ============ -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation (tested versions are from 1.10 to 1.12, see here_ for TF installation instructions). +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation (tested versions are from 1.2 to 1.11, see here_ for TF installation instructions). .. _here: https://www.tensorflow.org/install/ From 1903f5b4654aaf56d1ce00eaa54e563c1ab2d367 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 19:46:57 +0000 Subject: [PATCH 207/233] fix outdated file names --- docs/benchmark/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/benchmark/README.md b/docs/benchmark/README.md index ca379e28..33d24cc3 100644 --- a/docs/benchmark/README.md +++ b/docs/benchmark/README.md @@ -2,9 +2,9 @@ To benchmark the library, use the following commands ```bash # Running on CPU. -CUDA_VISIBLE_DEVICES= python profile.py --file_path logs_cpu.pkl +CUDA_VISIBLE_DEVICES= python benchmark.py --file_path logs_cpu.pkl # Running on GPU. -CUDA_VISIBLE_DEVICES=0 python profile.py --file_path logs_gpu.pkl +CUDA_VISIBLE_DEVICES=0 python benchmark.py --file_path logs_gpu.pkl ``` To visualize the results in a table, see ```results.ipynb``` Jupyter notebook. Here are the numbers you can get on NVIDIA DGX-1 server with Tesla V100 GPU and Intel(R) Xeon(R) CPU E5-2698 v4 @ 2.20GHz with 80 logical cores @@ -15,5 +15,5 @@ Here are the numbers you can get on NVIDIA DGX-1 server with Tesla V100 GPU and ## Comparing against TTPY To benchmark T3F against another library for Tensor Train decomposition [TTPY](https://github.com/oseledets/ttpy), install TTPY and run the following command in the bash shell ```bash -python profile_ttpy.py --file_path logs_ttpy.py +python benchmark_ttpy.py --file_path logs_ttpy.py ``` From 38b394f88f3ce391924a83590231b5fd72d06bf2 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 19:47:49 +0000 Subject: [PATCH 208/233] fix broken english --- docs/benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/benchmark/README.md b/docs/benchmark/README.md index 33d24cc3..cf5cc255 100644 --- a/docs/benchmark/README.md +++ b/docs/benchmark/README.md @@ -10,7 +10,7 @@ To visualize the results in a table, see ```results.ipynb``` Jupyter notebook. Here are the numbers you can get on NVIDIA DGX-1 server with Tesla V100 GPU and Intel(R) Xeon(R) CPU E5-2698 v4 @ 2.20GHz with 80 logical cores -[More details](https://t3f.readthedocs.io/en/latest/benchmark.html) on how to interpret this numbers see. +See [here](https://t3f.readthedocs.io/en/latest/benchmark.html) for a bit more details on how to interpret these numbers. ## Comparing against TTPY To benchmark T3F against another library for Tensor Train decomposition [TTPY](https://github.com/oseledets/ttpy), install TTPY and run the following command in the bash shell From ad2d987bc7426f46ddd13b1308b88a0c892d83b2 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 20:28:23 +0000 Subject: [PATCH 209/233] remove comparison notes that are repeated in the docs --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d5a63158..e22c609d 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,8 @@ TensorFlow implementation of a library for working with Tensor Train (TT) decomp The documentation is available via [readthedocs](https://t3f.readthedocs.io/en/latest/index.html). # Comparison with other libraries -There are implementations of the TT-toolbox in [plain Python](https://github.com/oseledets/ttpy) and [Matlab](https://github.com/oseledets/TT-Toolbox). Also, there is a very nice generic tensor network library [tnttorch](https://github.com/rballester/tntorch) which supports TT as a special case. - -The main difference between `t3f` Python/Matlab implementations is that `t3f` uses TensorFlow as backend and thus supports GPUs, automatic differentiation, and batch processing. As a more generic library, `tnttorch` lacks some TensorTrain specific tools (e.g. Riemannian optimization support). - -Here are the results im ms of benchmarking T3F on CPU and GPU and comparing against the [TTPY library](https://github.com/oseledets/ttpy) - - -For more details see ```examples/profile``` folder. +There are about a dozen other libraries implementing Tensor Train decomposition. +The main difference between `t3f` and other libraries is that `t3f` has extensive support for Riemannian optimization and that it uses TensorFlow as backend and thus supports GPUs, automatic differentiation, and batch processing. For a more detailed comparison with other libraries, see the [corresponding page](https://t3f.readthedocs.io/en/latest/comparison.html) in the docs. # Tests ```bash From 9e0228287ec7f28b61933709dde27c331c35ac78 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 20:30:48 +0000 Subject: [PATCH 210/233] API doc -> doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e22c609d..e68788f4 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The main difference between `t3f` and other libraries is that `t3f` has extensiv nosetests --logging-level=WARNING ``` -# Building API documentation +# Building documentation The documentation is build by sphinx and hosted on readthedocs.org. To locally rebuild the documentation, install sphinx and compile the docs by ```bash cd docs From df07ae148eaf7c4dfc4cf7e0ac56a1fdaaaf5e7f Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 20:43:02 +0000 Subject: [PATCH 211/233] update sphinx requirements --- docs/requirement.txt | 7 +++++++ docs/tf_requirement.txt | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 docs/requirement.txt delete mode 100644 docs/tf_requirement.txt diff --git a/docs/requirement.txt b/docs/requirement.txt new file mode 100644 index 00000000..15879c6a --- /dev/null +++ b/docs/requirement.txt @@ -0,0 +1,7 @@ +# We do not include tensorflow into pip requirements in the main module, but +# tf is necessary for building the docs with readthedocs.org: they fetch a fresh +# version of the library on each build and it doesn't import properly without +# tensorflow being installed. +tensorflow>=1.10,<=1.11 +ipykernel +nbsphinx diff --git a/docs/tf_requirement.txt b/docs/tf_requirement.txt deleted file mode 100644 index 8a20c945..00000000 --- a/docs/tf_requirement.txt +++ /dev/null @@ -1,7 +0,0 @@ -# This file is needed since we do not include tensorflow into pip requirements -# in the main module (since tf comes in many different forms and it's hard -# to check if one of them is installed), but tf is necessary for building -# the docs with readthedocs.org: they fetch a fresh version of the -# library on each build and it doesn't import properly without tensorflow -# being installed. -tensorflow>=1.0,<=1.5 From bf8adbedc1f6cc8bbfc83129aed9599cd50cd6c8 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 20:54:50 +0000 Subject: [PATCH 212/233] remove outdated comment about old python --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 68cc66c6..6310ecb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ env: - TF_VERSION=1.11 # command to install dependencies install: - # Python 2.7 has very old pip by default that doesn't have tensorflow. - pip install numpy tensorflow==$TF_VERSION keras - pip install coveralls # command to run tests From e125c22a4c67870d16457edda589bd03968a6dc9 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 20:56:32 +0000 Subject: [PATCH 213/233] add tf1.12 to the tests and docs --- .travis.yml | 2 +- docs/installation.rst | 2 +- docs/requirement.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6310ecb3..d4e04d94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ env: matrix: - TF_VERSION=1.2 - TF_VERSION=1.10 - - TF_VERSION=1.11 + - TF_VERSION=1.12 # command to install dependencies install: - pip install numpy tensorflow==$TF_VERSION keras diff --git a/docs/installation.rst b/docs/installation.rst index a2a1ade8..7ec3eb9d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -3,7 +3,7 @@ Installation ============ -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation (tested versions are from 1.2 to 1.11, see here_ for TF installation instructions). +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation (tested versions are from 1.2 to 1.12, see here_ for TF installation instructions). .. _here: https://www.tensorflow.org/install/ diff --git a/docs/requirement.txt b/docs/requirement.txt index 15879c6a..bb83c43b 100644 --- a/docs/requirement.txt +++ b/docs/requirement.txt @@ -2,6 +2,6 @@ # tf is necessary for building the docs with readthedocs.org: they fetch a fresh # version of the library on each build and it doesn't import properly without # tensorflow being installed. -tensorflow>=1.10,<=1.11 +tensorflow>=1.10,<=1.12 ipykernel nbsphinx From 033ce69eaff5cf713017af08b61d1df1583aa33e Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 22:10:03 +0000 Subject: [PATCH 214/233] Fix a typo in a name --- docs/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 973f8ca6..5fc153d1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,8 +56,8 @@ # General information about the project. project = u't3f' -copyright = u'2017, Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Mikhail Trofimov, Valentin Khrulkov' -author = u'Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Mikhail Trofimov, Valentin Khrulkov' +copyright = u'2017, Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Michael Figurnov, Valentin Khrulkov' +author = u'Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Michael Figurnov, Valentin Khrulkov' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -137,7 +137,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 't3f.tex', u't3f Documentation', - u'Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Mikhail Trofimov, Mikhail Trofimov, Valentin Khrulkov', 'manual'), + u'Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Michael Figurnov, Valentin Khrulkov', 'manual'), ] From 4722bcbd6df495443475e09830a263e9fa8f9059 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 6 Jan 2019 22:11:38 +0000 Subject: [PATCH 215/233] Fix a typo in a name --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index ebfe71a2..30a7786d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ -Copyright 2017 Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Mikhail Trofimov, Valentin Khrulkov +Copyright 2017 Alexander Novikov, Pavel Izmailov, Ivan Oseledets, Michael Figurnov, Valentin Khrulkov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From e4ade0628e420a5e1a7bd269da4143160ab84a2d Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 13 Jan 2019 12:37:20 +0000 Subject: [PATCH 216/233] Temporary add keras to docs requirements Otherwise API doesn't compile --- docs/requirement.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirement.txt b/docs/requirement.txt index bb83c43b..0d0164e5 100644 --- a/docs/requirement.txt +++ b/docs/requirement.txt @@ -3,5 +3,6 @@ # version of the library on each build and it doesn't import properly without # tensorflow being installed. tensorflow>=1.10,<=1.12 +keras ipykernel nbsphinx From 20a07efab7b64b7cb7fa609cc313914381e042e0 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 13 Jan 2019 12:41:04 +0000 Subject: [PATCH 217/233] Make link non-bold --- docs/quick_start.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quick_start.ipynb b/docs/quick_start.ipynb index 0eeceb70..fd0459c9 100644 --- a/docs/quick_start.ipynb +++ b/docs/quick_start.ipynb @@ -6,7 +6,7 @@ "source": [ "# Quick start\n", "\n", - "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/quick_start.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", + "[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/quick_start.ipynb) **this page in an interactive mode via Google Colaboratory.**\n", "\n", "In this quick starting guide we show the basics of working with t3f library. The main concept of the library is a TensorTrain object -- a compact (factorized) representation of a tensor (=multidimensional array). This is generalization of the matrix low-rank decomposition.\n", "\n", From 6cf2875a207b27a5a701f9bb4d9c1bb07c0a8011 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 13 Jan 2019 13:06:14 +0000 Subject: [PATCH 218/233] make links non-bold in other tutorials as well --- docs/tutorials/riemannian.ipynb | 2 +- docs/tutorials/tensor_completion.ipynb | 2 +- docs/tutorials/tensor_nets.ipynb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/riemannian.ipynb b/docs/tutorials/riemannian.ipynb index 28814e52..f2283fb3 100644 --- a/docs/tutorials/riemannian.ipynb +++ b/docs/tutorials/riemannian.ipynb @@ -6,7 +6,7 @@ "source": [ "# Riemannian optimization\n", "\n", - "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/riemannian.ipynb)** **this page in an interactive mode via Google Colaboratory.**" + "[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/riemannian.ipynb) **this page in an interactive mode via Google Colaboratory.**" ] }, { diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index 4a1667c0..da433839 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -6,7 +6,7 @@ "source": [ "# Tensor completion (example of minimizing a loss w.r.t. TT-tensor)\n", "\n", - "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_completion.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", + "[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_completion.ipynb) **this page in an interactive mode via Google Colaboratory.**\n", "\n", "In this example we will see how can we do tensor completion with t3f, i.e. observe a fraction of values in a tensor and recover the rest by assuming that the original tensor has low TT-rank.\n", "Mathematically it means that we have a binary mask $P$ and a ground truth tensor $A$, but we observe only a noisy and sparsified version of $A$: $P \\odot (\\hat{A})$, where $\\odot$ is the elementwise product (applying the binary mask) and $\\hat{A} = A + \\text{noise}$. In this case our task reduces to the following optimization problem:\n", diff --git a/docs/tutorials/tensor_nets.ipynb b/docs/tutorials/tensor_nets.ipynb index 929f60b8..1c75f385 100644 --- a/docs/tutorials/tensor_nets.ipynb +++ b/docs/tutorials/tensor_nets.ipynb @@ -23,7 +23,7 @@ "source": [ "# Tensor Nets (compressing neural networks)\n", "\n", - "**[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb)** **this page in an interactive mode via Google Colaboratory.**\n", + "[Open](https://colab.research.google.com/github/Bihaqo/t3f/blob/develop/docs/tutorials/tensor_nets.ipynb) **this page in an interactive mode via Google Colaboratory.**\n", "\n", "In this notebook we provide an example of how to build a simple Tensor Net (see https://arxiv.org/abs/1509.06569).\n", "\n", From 6e5ca6f1c1732581cddba4613955f745386c59d0 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 13 Jan 2019 13:09:30 +0000 Subject: [PATCH 219/233] use consistent subtitles type to fix table of contents --- docs/tutorials/tensor_completion.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index da433839..c5fb56b2 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -107,7 +107,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## SGD optimization\n", + "SGD optimization\n", + "-------------------------\n", "The simplest way to solve the optimization problem is Stochastic Gradient Descent: let TensorFlow differentiate the loss w.r.t. the factors (cores) of the TensorTrain decomposition of the estimated tensor and minimize the loss with your favourite SGD variation." ] }, @@ -197,7 +198,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Speeding it up\n", + "Speeding it up\n", + "--------------------\n", "The simple solution we have so far assumes that loss is computed by materializing the full estimated tensor and then zeroing out unobserved elements. If the tensors are really large and the fraction of observerd values is small (e.g. less than 1%), it may be much more efficient to directly work only with the observed elements." ] }, From cf49c0e19a2d625868c9ae9b1671284e51f6ee81 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 13 Jan 2019 13:14:57 +0000 Subject: [PATCH 220/233] check what happens if remove tensor_completion from index --- docs/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 226d4d8d..c7a241e4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,7 +25,6 @@ t3f is implemented on top of TensorFlow which gives it a few nice properties: :caption: Tutorials tutorials/tensor_nets - tutorials/tensor_completion tutorials/riemannian Citation From 55ebe951abfb76f7771e618a8e18262987dd7798 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 13 Jan 2019 13:18:11 +0000 Subject: [PATCH 221/233] try removing suspicious equation --- docs/index.rst | 1 + docs/tutorials/tensor_completion.ipynb | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index c7a241e4..226d4d8d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,6 +25,7 @@ t3f is implemented on top of TensorFlow which gives it a few nice properties: :caption: Tutorials tutorials/tensor_nets + tutorials/tensor_completion tutorials/riemannian Citation diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index c5fb56b2..3a86ecc7 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -10,14 +10,6 @@ "\n", "In this example we will see how can we do tensor completion with t3f, i.e. observe a fraction of values in a tensor and recover the rest by assuming that the original tensor has low TT-rank.\n", "Mathematically it means that we have a binary mask $P$ and a ground truth tensor $A$, but we observe only a noisy and sparsified version of $A$: $P \\odot (\\hat{A})$, where $\\odot$ is the elementwise product (applying the binary mask) and $\\hat{A} = A + \\text{noise}$. In this case our task reduces to the following optimization problem:\n", - "\\begin{equation*}\n", - "\\begin{aligned}\n", - "& \\underset{X}{\\text{minimize}} \n", - "& & \\|P \\odot (X - \\hat{A})\\|_F^2 \\\\\n", - "& \\text{subject to} \n", - "& & \\text{tt_rank}(X) \\leq r_0\n", - "\\end{aligned}\n", - "\\end{equation*}\n", "\n" ] }, From 445e3909b5b7c93ce2f54822f6a772d8832b41ef Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 13 Jan 2019 13:25:41 +0000 Subject: [PATCH 222/233] test other variations of equation --- docs/tutorials/tensor_completion.ipynb | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index 3a86ecc7..5348ec01 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -13,6 +13,45 @@ "\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\\begin{equation*}\n", + "\\begin{aligned}\n", + "& \\underset{X}{\\text{minimize}} \n", + "& & \\|P \\odot (X - \\hat{A})\\|_F^2 \\\\\n", + "& \\text{subject to} \n", + "& & \\text{tt_rank}(X) \\leq r_0\n", + "\\end{aligned}\n", + "\\end{equation*}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\\begin{equation*}\n", + "\\begin{aligned}\n", + "5 + 5\n", + "\\end{aligned}\n", + "\\end{equation*}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\\begin{equation*}\n", + "\\begin{aligned}\n", + "& \\text{minimize}\n", + "& & \\|P \\odot (X - \\hat{A})\\|_F^2 \\\\\n", + "& \\text{subject to} \n", + "& & \\text{ttrank}(X) \\leq r_0\n", + "\\end{aligned}\n", + "\\end{equation*}" + ] + }, { "cell_type": "code", "execution_count": 1, From f9afd71e051786310e0f66480dca1abe77bd4f72 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Sun, 13 Jan 2019 13:29:10 +0000 Subject: [PATCH 223/233] try with $ instead of begin{equation} --- docs/tutorials/tensor_completion.ipynb | 37 +++----------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/docs/tutorials/tensor_completion.ipynb b/docs/tutorials/tensor_completion.ipynb index 5348ec01..a65b3154 100644 --- a/docs/tutorials/tensor_completion.ipynb +++ b/docs/tutorials/tensor_completion.ipynb @@ -10,46 +10,15 @@ "\n", "In this example we will see how can we do tensor completion with t3f, i.e. observe a fraction of values in a tensor and recover the rest by assuming that the original tensor has low TT-rank.\n", "Mathematically it means that we have a binary mask $P$ and a ground truth tensor $A$, but we observe only a noisy and sparsified version of $A$: $P \\odot (\\hat{A})$, where $\\odot$ is the elementwise product (applying the binary mask) and $\\hat{A} = A + \\text{noise}$. In this case our task reduces to the following optimization problem:\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\\begin{equation*}\n", + "\n", + "$$\n", "\\begin{aligned}\n", "& \\underset{X}{\\text{minimize}} \n", "& & \\|P \\odot (X - \\hat{A})\\|_F^2 \\\\\n", "& \\text{subject to} \n", "& & \\text{tt_rank}(X) \\leq r_0\n", "\\end{aligned}\n", - "\\end{equation*}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\\begin{equation*}\n", - "\\begin{aligned}\n", - "5 + 5\n", - "\\end{aligned}\n", - "\\end{equation*}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\\begin{equation*}\n", - "\\begin{aligned}\n", - "& \\text{minimize}\n", - "& & \\|P \\odot (X - \\hat{A})\\|_F^2 \\\\\n", - "& \\text{subject to} \n", - "& & \\text{ttrank}(X) \\leq r_0\n", - "\\end{aligned}\n", - "\\end{equation*}" + "$$" ] }, { From 2b6aed9484f51f0dab5afdd3a7466ae1d0ad3a61 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 23 Sep 2019 14:00:56 +0100 Subject: [PATCH 224/233] Try updating tf version to the latest ones --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4e04d94..84cd8729 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,9 @@ python: - "3.6" env: matrix: - - TF_VERSION=1.2 - - TF_VERSION=1.10 - - TF_VERSION=1.12 + - TF_VERSION=1.12.1 + - TF_VERSION=1.13 + - TF_VERSION=1.14 # command to install dependencies install: - pip install numpy tensorflow==$TF_VERSION keras From c834d22cfbf8dd5913a74c90d3ddcf8d87bb77a3 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 23 Sep 2019 14:28:20 +0100 Subject: [PATCH 225/233] Use existing tf versions --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 84cd8729..32eebdf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ python: - "3.6" env: matrix: - - TF_VERSION=1.12.1 - - TF_VERSION=1.13 + - TF_VERSION=1.12.2 + - TF_VERSION=1.13.2 - TF_VERSION=1.14 # command to install dependencies install: From e237aeae2420c5f370b7c3870c87762ad0a9b35c Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 23 Sep 2019 14:44:02 +0100 Subject: [PATCH 226/233] try removing some of the dependencies which should be there automatically --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 32eebdf9..9e0f561d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ env: - TF_VERSION=1.14 # command to install dependencies install: - - pip install numpy tensorflow==$TF_VERSION keras + - pip install tensorflow==$TF_VERSION - pip install coveralls # command to run tests script: From 78117e4c4a4853e758a8fafd6131a3d1b4fb812a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 23 Sep 2019 14:50:57 +0100 Subject: [PATCH 227/233] import keras from tf and update docs requirements --- docs/requirement.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/requirement.txt b/docs/requirement.txt index 0d0164e5..26674f8b 100644 --- a/docs/requirement.txt +++ b/docs/requirement.txt @@ -2,7 +2,6 @@ # tf is necessary for building the docs with readthedocs.org: they fetch a fresh # version of the library on each build and it doesn't import properly without # tensorflow being installed. -tensorflow>=1.10,<=1.12 -keras +tensorflow>=1.12,<=1.14 ipykernel nbsphinx From 3add9f8aa578ab292300d2e83e526315ffe3bd80 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Mon, 23 Sep 2019 15:02:31 +0100 Subject: [PATCH 228/233] forgot to add a bit: now actually import keras from tf --- t3f/nn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/nn.py b/t3f/nn.py index 1d8e3ebb..f96a480d 100644 --- a/t3f/nn.py +++ b/t3f/nn.py @@ -2,8 +2,8 @@ from itertools import count import numpy as np -from keras.engine.topology import Layer -from keras.layers import Activation +from tensorflow.keras.layers import Layer +from tensorflow.keras.layers import Activation import t3f import tensorflow as tf From 018d3ab38632f062c127e88afa133c46fbd3c291 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 24 Sep 2019 17:18:08 +0100 Subject: [PATCH 229/233] Fix outdated keras bit --- t3f/nn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t3f/nn.py b/t3f/nn.py index f96a480d..ceb9a378 100644 --- a/t3f/nn.py +++ b/t3f/nn.py @@ -67,9 +67,9 @@ def build(self, input_shape): b_init = tf.constant_initializer(self.bias_initializer) self.b = tf.get_variable('bias', shape=self.output_dim, initializer=b_init) - self.trainable_weights = list(self.matrix.tt_cores) + self._trainable_weights = list(self.matrix.tt_cores) if self.b is not None: - self.trainable_weights.append(self.b) + self._trainable_weights.append(self.b) def call(self, x): res = t3f.matmul(x, self.matrix) From a819008c4c4caac6238740cd4556247b0ab83e19 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 24 Sep 2019 17:28:40 +0100 Subject: [PATCH 230/233] update docs to have the actual tested tf versions --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 7ec3eb9d..2410254e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -3,7 +3,7 @@ Installation ============ -T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation (tested versions are from 1.2 to 1.12, see here_ for TF installation instructions). +T3f assumes you have Python 2.7, 3.4, 3.5, or 3.6 and a working TensorFlow installation (tested versions are from 1.12 to 1.14, see here_ for TF installation instructions). .. _here: https://www.tensorflow.org/install/ From 1fd08f18e0e37c2b379ff7ce6e667b4e36dd5877 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 25 Sep 2019 17:31:23 +0100 Subject: [PATCH 231/233] s/quadratic_form/bilinear_form/ --- t3f/ops.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/t3f/ops.py b/t3f/ops.py index 3afd39df..a271b163 100644 --- a/t3f/ops.py +++ b/t3f/ops.py @@ -1036,8 +1036,15 @@ def transpose(tt_matrix, name='t3f_transpose'): batch_size) -def quadratic_form(A, b, c, name='t3f_quadratic_form'): - """Quadratic form b^t A c; A is a TT-matrix, b and c can be batches. +def quadratic_form(A, b, c, name='t3f_bilinear_form'): + """Outdated, see `bilinear_form`.""" + print('Warning: function quadratic_form is being depricated and ' + 'replaced with bilinear_form.') + return bilinear_form(A, b, c) + + +def bilinear_form(A, b, c, name='t3f_bilinear_form'): + """Bilinear form b^t A c; A is a TT-matrix, b and c can be batches. Args: A: `TensorTrain` object containing a TT-matrix of size N x M. @@ -1048,7 +1055,7 @@ def quadratic_form(A, b, c, name='t3f_quadratic_form'): name: string, name of the Op. Returns: - A number, the value of the quadratic form if all the arguments are + A number, the value of the bilinear form if all the arguments are `TensorTrain`s. OR tf.Tensor of size batch_size if at least one of the arguments is `TensorTrainBatch` From e6cb9fcd7e5f2412c3b24a15a45ebb4434d67c3f Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 25 Sep 2019 17:48:26 +0100 Subject: [PATCH 232/233] Update quadratic to bilinear everywhere else --- t3f/__init__.py | 1 + t3f/autodiff.py | 2 +- t3f/autodiff_test.py | 4 ++-- t3f/ops_test.py | 12 ++++++------ 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/t3f/__init__.py b/t3f/__init__.py index 05d318eb..1b57c500 100644 --- a/t3f/__init__.py +++ b/t3f/__init__.py @@ -14,6 +14,7 @@ from t3f.ops import matmul from t3f.ops import multiply from t3f.ops import quadratic_form +from t3f.ops import bilinear_form from t3f.ops import transpose from t3f.ops import gather_nd from t3f.ops import renormalize_tt_cores diff --git a/t3f/autodiff.py b/t3f/autodiff.py index a93b3e20..038b7a51 100644 --- a/t3f/autodiff.py +++ b/t3f/autodiff.py @@ -131,7 +131,7 @@ def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product', # It's Riemannian Hessian by vector product is # proj_vec = t3f.project(vector, x) # t3f.project(t3f.matmul(A + t3f.transpose(A), proj_vec), x) - f = lambda x: t3f.quadratic_form(A, x, x) + f = lambda x: t3f.bilinear_form(A, x, x) res = t3f.hessian_vector_product(f, x, vector) Args: diff --git a/t3f/autodiff_test.py b/t3f/autodiff_test.py index 257cef8f..360fbd89 100644 --- a/t3f/autodiff_test.py +++ b/t3f/autodiff_test.py @@ -29,7 +29,7 @@ def func1(x): self._TestSingleGradient(func1, x, desired1) def func2(x): - return ops.quadratic_form(A, x, x) + return ops.bilinear_form(A, x, x) grad = ops.matmul(ops.transpose(A) + A, x) desired2 = ops.full(riemannian.project(grad, x)) self._TestSingleGradient(func2, x, desired2) @@ -71,7 +71,7 @@ def func1(x): self._TestSingleHessianByVector(func1, x, z, desired1) def func2(x): - return ops.quadratic_form(A, x, x) + return ops.bilinear_form(A, x, x) # Hessian of is A + A.T hessian_by_vector = ops.matmul(ops.transpose(A) + A, projected_vector) desired2 = ops.full(riemannian.project(hessian_by_vector, x)) diff --git a/t3f/ops_test.py b/t3f/ops_test.py index 964262d1..8e34bb07 100644 --- a/t3f/ops_test.py +++ b/t3f/ops_test.py @@ -357,8 +357,8 @@ def testTranspose(self): res_actual_val, tt_val = sess.run([res_actual, ops.full(tt)]) self.assertAllClose(tt_val.transpose(), res_actual_val) - def testQuadraticForm(self): - # Test quadratic form. + def testBilinearForm(self): + # Test bilinear form. shape_list = (((2, 2), (3, 4)), ((2, 3, 4), (2, 2, 2))) rank_list = (1, 2) @@ -371,15 +371,15 @@ def testQuadraticForm(self): dtype=self.dtype) c = initializers.random_matrix((tensor_shape[1], None), tt_rank=rank, dtype=self.dtype) - res_actual = ops.quadratic_form(A, b, c) + res_actual = ops.bilinear_form(A, b, c) vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] res_actual_val, A_val, b_val, c_val = sess.run(vars) res_desired = b_val.T.dot(A_val).dot(c_val) self.assertAllClose(res_actual_val, np.squeeze(res_desired), atol=1e-5, rtol=1e-5) - def testQuadraticFormBatch(self): - # Test quadratic form for batch of tensors. + def testBilinearFormBatch(self): + # Test bilinear form for batch of tensors. shape_list = (((2, 2), (3, 4)), ((2, 3, 4), (2, 2, 2))) rank_list = (1, 2) @@ -394,7 +394,7 @@ def testQuadraticFormBatch(self): c = initializers.random_matrix_batch((tensor_shape[1], None), tt_rank=rank, batch_size=5, dtype=self.dtype) - res_actual = ops.quadratic_form(A, b, c) + res_actual = ops.bilinear_form(A, b, c) vars = [res_actual, ops.full(A), ops.full(b), ops.full(c)] res_actual_val, A_val, b_val, c_val = sess.run(vars) res_desired = np.diag(b_val[:, :, 0].dot(A_val).dot(c_val[:, :, 0].T)) From 7497a9aa7d7f191b60d2d538d9ba11f6a96fa6ee Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 22 Oct 2019 23:29:22 +0100 Subject: [PATCH 233/233] changelog and new version number --- CHANGELOG.md | 21 +++++++++++++++++++++ setup.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 048df107..0f8df0e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +## [1.1.0] - 2019-10-22 +### Added +- Proper documentation with tutorials. +- Benchmarks. +- Asymptotic complexities of some function in the docstrings. +- Eager mode support. +- Ops have name scopes now. +- Riemannian autodiff (Yey!) +- Better support for using dtypes other than float32. +- Neural network module. + +### Changed +- Change to support TF 1.12-1.14 instead of TF 1.0-1.4 +- Bug fixes. +- Bug fix in benchmarks. +- Rename quadratic_form to bilinear_form. +- Better test coverage. + ## [1.0.0] - 2018-01-08 ### Added - API reference documentation. @@ -73,5 +91,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - frobenius_norm [Unreleased]: https://github.com/Bihaqo/t3f/compare/master...develop +[1.1.0]: https://github.com/Bihaqo/t3f/compare/1.0.0...1.1.0 +[1.0.0]: https://github.com/Bihaqo/t3f/compare/0.3.0...1.0.0 +[0.3.0]: https://github.com/Bihaqo/t3f/compare/0.2.0...0.3.0 [0.2.0]: https://github.com/Bihaqo/t3f/compare/0.1.0...0.2.0 [0.1.0]: https://github.com/Bihaqo/t3f/compare/f24409508...0.1.0 diff --git a/setup.py b/setup.py index fcdfdb76..5b165d43 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup setup(name='t3f', - version='1.0.0', + version='1.1.0', description='Tensor Train decomposition on TensorFlow', url='https://github.com/Bihaqo/t3f', author='Alexander Novikov',