-
Notifications
You must be signed in to change notification settings - Fork 5
/
get-zotero-data.xql
237 lines (225 loc) · 11.3 KB
/
get-zotero-data.xql
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
xquery version "3.1";
(:~
: XQuery Zotero integration
: Queries Zotero API : https://api.zotero.org
: Checks for updates since last modified version using Zotero Last-Modified-Version header
: Converts Zotero records to Syriaca.org TEI using zotero2tei.xqm
: Adds new records to directory.
:
: To be done:
: Submit to Perseids
:)
import module namespace http="http://expath.org/ns/http-client";
import module namespace zotero2tei="http://syriaca.org/zotero2tei" at "zotero2tei.xqm";
import module namespace console="http://exist-db.org/xquery/console";
declare namespace tei="http://www.tei-c.org/ns/1.0";
declare namespace output = "http://www.w3.org/2010/xslt-xquery-serialization";
declare variable $zotero-api := 'https://api.zotero.org';
(: Access zotero-api configuration file :)
declare variable $zotero-config := doc('zotero-config.xml');
(: Zotero group id :)
declare variable $groupid := $zotero-config//groupid/text();
(: Zotero last modified version, to check for updates. :)
declare variable $last-modified-version := $zotero-config//last-modified-version/text();
(: Directory bibl data is stored in :)
declare variable $data-dir := $zotero-config//data-dir/text();
(: Local URI pattern for bibl records :)
declare variable $base-uri := $zotero-config//base-uri/text();
(: Format defaults to tei :)
declare variable $format := if($zotero-config//format/text() != '') then $zotero-config//format/text() else 'tei';
(:~
: Convert records to Syriaca.org compliant TEI records, using zotero2tei.xqm
: Save records to the database.
: @param $record
: @param $index-number
: @param $format
:)
declare function local:process-records($record as item()?, $format as xs:string?){
let $idNumber :=
if($zotero-config//id-pattern/text() = 'zotero' and $format = 'json') then
tokenize($record?links?alternate?href,'/')[last()]
else if($zotero-config//id-pattern/text() = 'zotero' and $format ='tei') then
tokenize($record/@corresp,'/')[last()]
else util:random(10000)
(:let $id := local:make-local-uri($index-number):)
let $file-name := concat($idNumber,'.xml')
let $new-record := zotero2tei:build-new-record($record, $idNumber, $format)
return
if($idNumber != '') then
try {xmldb:store($data-dir, xmldb:encode-uri($file-name), $new-record)} catch *{
<response status="fail">
<message>Failed to add resource {$file-name}: {concat($err:code, ": ", $err:description), console:log(concat($err:code, ": ", $err:description))}</message>
</response>
}
else ()
};
(:~
: Get highest existing local id in the eXist database. Increment new record ids
: @param $path to existing bibliographic data
: @param $base-uri base uri defined in repo.xml, establishing pattern for bibl ids. example: http://syriaca.org/bibl
:)
declare function local:make-local-uri($index-number as xs:integer) {
let $all-bibl-ids :=
for $uri in collection($data-dir)/tei:TEI/tei:text/tei:body/tei:biblStruct/descendant::tei:idno[starts-with(.,$base-uri)]
return number(replace(replace($uri,$base-uri,''),'/tei',''))
let $max := max($all-bibl-ids)
return
if($max) then concat($base-uri,'/', ($max + $index-number))
else concat($base-uri,'/',$index-number)
};
(:~
: Update stored last modified version (from Zotero API) in zotero-config.xml
:)
declare function local:update-version($version as xs:string?) {
try {
<response status="200">
<message>{for $v in $zotero-config//last-modified-version return update value $v with $version}</message>
</response>
} catch *{
<response status="fail">
<message>Failed to update last-modified-version: {concat($err:code, ": ", $err:description)}</message>
</response>
}
};
(:~
: Page through Zotero results
: @param $groupid
: @param $last-modified-version
: @param $total
: @param $start
: @param $perpage
:)
declare function local:process-results($results as item()*){
let $items := $results
let $headers := $items[1]
let $results :=
if($format = 'json') then
parse-json(util:binary-to-string($items[2]))
else $items[2]
return
if($headers/@status = '200') then
if($format = 'json') then
for $rec at $p in $results?*
where not(exists($rec?data?parentItem))
return local:process-records($rec, $format)
else
for $rec at $p in $results//tei:biblStruct
return $rec(:local:process-records($rec, $format):)
(: else if($headers/@name="Backoff") then:)
(: (<message status="{$headers/@status}">{string($headers/@message)}</message>,:)
(: let $wait := util:wait(xs:integer($headers[@name="Backoff"][@value])):)
(: return local:get-zotero():)
(: ):)
(: else if($headers/@name="Retry-After") then :)
(: (<message status="{$headers/@status}">{string($headers/@message)}</message>,:)
(: let $wait := util:wait(xs:integer($headers[@name="Retry-After"][@value])):)
(: return local:get-zotero():)
(: ):)
else <message status="{$headers/@status}">{string($headers/@message)}</message>
};
(:~
: Get Zotero data
: Check for updates since last modified version (stored in $zotero-config)
: @param $groupid Zotero group id
: @param $last-modified-version
:)
declare function local:get-zotero-data($url){
http:send-request(<http:request http-version="1.1" href="{xs:anyURI($url)}" method="get"/>)
(:
if(request:get-parameter('action', '') = 'initiate') then
http:send-request(<http:request http-version="1.1" href="{xs:anyURI($url)}" method="get"/>)
else http:send-request(<http:request http-version="1.1" href="{xs:anyURI($url)}" method="get">
<http:header name="If-Modified-Since-Version" value="{$last-modified-version}"/>
</http:request>)
:)
};
(:~
: Get and process Zotero data.
:)
declare function local:get-zotero(){
let $start := if(request:get-parameter('start', '') != '') then concat('&start=',request:get-parameter('start', '')) else '&start=0'
let $limit := if(request:get-parameter('limit', '') != '') then concat('&limit=',request:get-parameter('limit', '')) else '&limit=50'
let $action := if(request:get-parameter('action', '') != '') then request:get-parameter('action', '') else 'check'
let $since := if($action = 'update') then concat('&since=',$last-modified-version) else ()
let $url := if(request:get-parameter('next', '') != '') then request:get-parameter('next', '') else concat($zotero-api,'/groups/',$groupid,'/items?format=',$format,if($format='json') then '&include=bib,data,coins,citation&style=chicago-fullnote-bibliography' else(),$start, $limit, $since)
let $items := local:get-zotero-data($url)
let $items-info := $items[1]
let $total := $items-info/http:header[@name='total-results']/@value
let $version := $items-info/http:header[@name='last-modified-version']/@value
let $links := string($items-info/http:header[@name='link']/@value)
let $next-link := tokenize($links,',')[contains(., 'rel="next"')]
let $next-url := replace(substring-before($next-link, '; '),'<|>','')
let $new-start := substring-after($next-url,'start=')
return
if(request:get-parameter('action', '') = 'check') then
<div xmlns="http://www.w3.org/1999/xhtml" id="response">
{
if($items-info/@message="Not Modified") then
<p><label>Updates : </label> No updates available.</p>
else
(<p><label>Last Modified Version (Zotero): </label> {string($version)}</p>,
<p><label>Number of updated records: </label> {string($total)}</p>
)
}
</div>
else if($items-info/@status = '200') then
<div xmlns="http://www.w3.org/1999/xhtml" id="response">
{
let $results :=
(local:process-results($items),
if($next-url) then () else local:update-version($version))
return
(if(request:get-parameter('debug', '') = 'true') then $results else (),
if($next-url) then
<div xmlns="http://www.w3.org/1999/xhtml">
<p>Processed {if(request:get-parameter('start', '') != '') then request:get-parameter('start', '') else '0'} - {substring-before($new-start,'&')} of {string($total)}</p>
<p><a href="get-zotero-data.xql?action={$action}&start={$new-start}{$since}" class="btn btn-info zotero">Next</a></p>
</div>
else
if($items-info/@message="Not Modified") then
<p><label>Updates : </label> No updates available.</p>
else
<div><h3>Updated</h3>
<p><label>Last Modified Version (Zotero): </label> {string($version)}</p>
<p><label>Number of updated records: </label> {string($total)}</p>
</div>)
}</div>
else <message status="{$items-info/@status}">{string($items-info/@message)} {$url}</message>
};
(: Helper function to recursively create a collection hierarchy. :)
declare function local:mkcol-recursive($collection, $components) {
if (exists($components)) then
let $newColl := concat($collection, "/", $components[1])
return (
xmldb:create-collection($collection, $components[1]),
local:mkcol-recursive($newColl, subsequence($components, 2))
)
else ()
};
(: Helper function to recursively create a collection hierarchy. :)
declare function local:mkcol($collection, $path) {
local:mkcol-recursive($collection, tokenize($path, "/"))
};
(:~
: Check action parameter, if empty, return contents of config.xml
: If $action is not empty, check for specified collection, create if it does not exist.
: Run Zotero request.
:)
if(request:get-parameter('action', '') = 'update') then
if(xmldb:collection-available($data-dir)) then
local:get-zotero()
else (local:mkcol("/db/apps", replace($data-dir,'/db/apps','')),local:get-zotero())
else if(request:get-parameter('action', '') = 'initiate') then
if(request:get-parameter('start', '') != '' and xmldb:collection-available($data-dir)) then
local:get-zotero()
else if((request:get-parameter('start', '') = '0' or request:get-parameter('start', '') = '1') and xmldb:collection-available($data-dir)) then
local:get-zotero()
else if(xmldb:collection-available($data-dir)) then
(xmldb:remove($data-dir),local:mkcol("/db/apps", replace($data-dir,'/db/apps','')),local:get-zotero())
else (local:mkcol("/db/apps", replace($data-dir,'/db/apps','')),local:get-zotero())
else
<div xmlns="http://www.w3.org/1999/xhtml">
<p><label>Group ID : </label> {$groupid}</p>
<p><label>Last Modified Version (Zotero): </label> {$last-modified-version}</p>
<p><label>Data Directory : </label> {$data-dir}</p>
</div>