New UI for prep stage

appveyor.yml

@@ -17,6 +17,7 @@ install:
   - "%PYTHON%\\python.exe -m conda install -q --yes cython scikit-learn pandas" # for vsm 
   - "%PYTHON%\\python.exe -c \"import nltk; nltk.download('stopwords'); nltk.download('punkt')\""
   - "%PYTHON%\\python.exe -m pip install unittest2 nose wget"
+  - "%PYTHON%\\python.exe -m pip install -e git+https://github.com/peterbrittain/asciimatics.git@fcedb4947933de7e1507ec0dee8ca7a3f466928a#egg=asciimatics"
   - "%PYTHON%\\python.exe -m pip install ."
 
 build: off

requirements.txt

@@ -1,3 +1,4 @@
+-e git+https://github.com/peterbrittain/asciimatics.git@fcedb4947933de7e1507ec0dee8ca7a3f466928a#egg=asciimatics
 bottle>=0.12.0
 brewer2mpl>=1.4.0,<1.5.0
 decorator>=4.0.5

tests/test_prep.py

@@ -58,50 +58,54 @@ def test_get_candidate_words():
 
 @patch('topicexplorer.prep.input')
 def test_get_high_filter(input_mock):
-    input_mock.side_effect = ['3', 'y']
-    high_filter, candidates = topicexplorer.prep.get_high_filter(corpus)
-    assert high_filter == 3
+    # Test with high filter of 3
+    items, counts = topicexplorer.prep.get_corpus_counts(corpus)
+    candidates, filtered, valid = topicexplorer.prep.get_high_filter_stops(corpus, words=set(), items=items, counts=counts, num=3)
+    assert len(corpus.words) - len(candidates) == 3
     assert candidates == ['I']
+    assert valid == True
 
-    # Test selection of all words
-    input_mock.side_effect = ['3', '1', '3', 'y']
-    high_filter, candidates = topicexplorer.prep.get_high_filter(corpus)
-    assert high_filter == 3
-    assert candidates == ['I']
+    # Test with high filter of 0
+    candidates, filtered, valid = topicexplorer.prep.get_high_filter_stops(corpus, words=set(), items=items, counts=counts, num=0)
+    assert len(corpus.words) - len(candidates) == 4
+    assert candidates == []
+    assert valid == True
 
-    # Test not accept
-    input_mock.side_effect = ['3', 'n', '3', 'y']
-    high_filter, candidates = topicexplorer.prep.get_high_filter(corpus)
-    assert high_filter == 3
-    assert candidates == ['I']
-
-    # Test invalid action
-    input_mock.side_effect = ['blahhhh', '3', 'y']
-    high_filter, candidates = topicexplorer.prep.get_high_filter(corpus)
-    assert high_filter == 3
-    assert candidates == ['I']
+    # Test with high filter of 1, should return invalid
+    candidates, filtered, valid = topicexplorer.prep.get_high_filter_stops(corpus, words=set(), items=items, counts=counts, num=1)
+    assert len(corpus.words) - len(candidates) == 0
+    assert candidates == ['I', 'came', 'conquered', 'saw']
+    assert valid == False
+
+    # Test with high filter of 100
+    candidates, filtered, valid = topicexplorer.prep.get_high_filter_stops(corpus, words=set(), items=items, counts=counts, num=100)
+    assert len(corpus.words) - len(candidates) == 4
+    assert candidates == []
+    assert valid == True
 
 @patch('topicexplorer.prep.input')
 def test_get_low_filter(input_mock):
-    input_mock.side_effect = ['1', 'y']
-    low_filter, candidates = topicexplorer.prep.get_low_filter(corpus)
-    assert low_filter == 1 
+    # Test with low filter of 1
+    items, counts = topicexplorer.prep.get_corpus_counts(corpus)
+    candidates, filtered, valid = topicexplorer.prep.get_low_filter_stops(corpus, words=set(), items=items, counts=counts, num=1)
+    assert len(corpus.words) - len(candidates) == 1 
     assert all(w in candidates for w in ['came', 'saw', 'conquered'])
+    assert valid == True
 
-    # Test selection of all words
-    input_mock.side_effect = ['1', '3', '1', 'y']
-    low_filter, candidates = topicexplorer.prep.get_low_filter(corpus)
-    assert low_filter == 1
-    assert all(w in candidates for w in ['came', 'saw', 'conquered'])
-
-    # Test not accept
-    input_mock.side_effect = ['1', 'n', '1', 'y']
-    low_filter, candidates = topicexplorer.prep.get_low_filter(corpus)
-    assert low_filter == 1
-    assert all(w in candidates for w in ['came', 'saw', 'conquered'])
-
-    # Test invalid action
-    input_mock.side_effect = ['blahhhh', '1', 'y']
-    low_filter, candidates = topicexplorer.prep.get_low_filter(corpus)
-    assert low_filter == 1
-    assert all(w in candidates for w in ['came', 'saw', 'conquered'])
+    # Test with low filter of 3
+    candidates, filtered, valid = topicexplorer.prep.get_low_filter_stops(corpus, words=set(), items=items, counts=counts, num=3)
+    assert len(corpus.words) - len(candidates) == 0
+    assert all(w in candidates for w in ['came', 'saw', 'conquered', 'I'])
+    assert valid == False
+
+    # Test with low filter of 0
+    candidates, filtered, valid = topicexplorer.prep.get_low_filter_stops(corpus, words=set(), items=items, counts=counts, num=0)
+    assert len(corpus.words) - len(candidates) == 4
+    assert all(w in candidates for w in [])
+    assert valid == True
+
+    # Test with low filter of 100
+    candidates, filtered, valid = topicexplorer.prep.get_low_filter_stops(corpus, words=set(), items=items, counts=counts, num=100)
+    assert len(corpus.words) - len(candidates) == 0
+    assert all(w in candidates for w in ['came', 'saw', 'conquered', 'I'])
+    assert valid == False