From 31b3889282d7e747b2d5c155644a0d6d9ea8cd19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Mon, 22 Aug 2016 18:34:00 +0200 Subject: [PATCH 1/4] Added support for ujson (if available) and added /pagedjson page which seperates the json files into pages. (default 50 items per page). With the "items" param you can specify how many items should be shown on a single page. With "page" you specify which page number to show. --- webui.py | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/webui.py b/webui.py index 15f326a..3732fc8 100755 --- a/webui.py +++ b/webui.py @@ -7,13 +7,20 @@ import datetime import glob import hashlib -import json import csv import StringIO import ConfigParser import string import shlex import urllib + +# use ujson if avalible (faster than built in json) +try: + import ujson as json +except ImportError: + import json + print("ujson module not found! Consider installing it for improved Json generating speed!") + # import recoll and rclextract try: from recoll import recoll @@ -150,6 +157,16 @@ def get_query(): 'page': int(select([bottle.request.query.get('page'), 0])), } return query + +#}}} +#{{{ get_pages +def get_pages(): + pages = { + 'page': select([bottle.request.query.get('page'), 1]), + 'items': select([bottle.request.query.get('items'), 50]), + } + return pages + #}}} #{{{ query_to_recoll_string def query_to_recoll_string(q): @@ -309,6 +326,48 @@ def edit(resnum): if pathismine: os.unlink(path) return f + + +#}}} +#{{{ recoll_search_pages +def recoll_search_page(q, item_per_page=50, page=1, dosnippets=True): + config = get_config() + tstart = datetime.datetime.now() + results = [] + query = recoll_initsearch(q) + nres = query.rowcount + + item_per_page = int(item_per_page) + page = int(page) + + offset = (page - 1) * item_per_page + try: + query.scroll(offset, mode='absolute') + except: + pass + + highlighter = HlMeths() + for i in range(item_per_page): + try: + doc = query.fetchone() + except: + break + d = {} + for f in FIELDS: + v = getattr(doc, f) + if v is not None: + d[f] = v.encode('utf-8') + else: + d[f] = '' + d['label'] = select([d['title'], d['filename'], '?'], [None, '']) + d['sha'] = hashlib.sha1(d['url']+d['ipath']).hexdigest() + d['time'] = timestr(d['mtime'], config['timefmt']) + if dosnippets: + d['snippet'] = query.makedocabstract(doc, highlighter).encode('utf-8') + results.append(d) + tend = datetime.datetime.now() + return results, nres, tend - tstart + #}}} #{{{ json @bottle.route('/json') @@ -321,6 +380,22 @@ def get_json(): res, nres, timer = recoll_search(query) return json.dumps({ 'query': query, 'results': res }) + + +#}}} +#{{{ pagedjson +@bottle.route('/pagedjson') +def get_page_json(): + query = get_query() + pages = get_pages() + query['page'] = pages['page'] + qs = query_to_recoll_string(query) + bottle.response.headers['Content-Type'] = 'application/json' + bottle.response.headers['Content-Disposition'] = 'attachment; filename=recoll-%s.json' % normalise_filename(qs) + res, nres, timer = recoll_search_page(query,pages['items'],pages['page']) + + return json.dumps({ 'query': query, 'results': res }) + #}}} #{{{ csv @bottle.route('/csv') From 244dda8383e82500cd04124aed50c3977f99531e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Wed, 24 Aug 2016 17:33:36 +0200 Subject: [PATCH 2/4] Update README.rst --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index bb3c4f9..6a7dd95 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,9 @@ Recoll WebUI ============ +This is an modified Version for use with the Recolldroid android app. It adds an feature to export only a "page" of results to json +and not all at once. + **Recoll WebUI** is a Python-based web interface for **Recoll** text search tool for Unix/Linux. From 155a820318ca754cf19ac81abbd2ce66adb071cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Wed, 24 Aug 2016 17:34:00 +0200 Subject: [PATCH 3/4] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6a7dd95..5d25b3c 100644 --- a/README.rst +++ b/README.rst @@ -2,8 +2,8 @@ Recoll WebUI ============ -This is an modified Version for use with the Recolldroid android app. It adds an feature to export only a "page" of results to json -and not all at once. +**This is an modified Version for use with the Recolldroid android app. It adds an feature to export only a "page" of results to json +and not all at once.** **Recoll WebUI** is a Python-based web interface for **Recoll** text search tool for Unix/Linux. From 7fed7c3f95de0b1549eef54d5067fdd5dd80411f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20B=C3=B6hmer?= Date: Wed, 24 Aug 2016 17:34:30 +0200 Subject: [PATCH 4/4] Update README.rst --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 5d25b3c..9936497 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -============ -Recoll WebUI -============ +============================ +Recoll WebUI for Recolldroid +============================ **This is an modified Version for use with the Recolldroid android app. It adds an feature to export only a "page" of results to json and not all at once.**