-
Notifications
You must be signed in to change notification settings - Fork 0
/
check_mt-otpt.py
199 lines (171 loc) · 7.43 KB
/
check_mt-otpt.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
# -*- coding:utf-8 -*-
# Script that takes a tmx file and for each translation unit <tu>
# takes the source string and translates into the target language.
# The output is a table with the following structure
# source_language original STRING
# target_language original STRING
# target_language Apertium translated STRING
#
# Usage:
# (1) python check_mt-otpt.py
# - the input dir is tmx_data
# (2) python check_mt-otpt.py -d DIRECTORY_NAME
# - the input dir is DIRECTORY_NAME
# (3) python check_mt-otpt.py -f FILE_NAME
# - the input file is FILE_NAME
#
# the output dir is 'otpt_dir' in the current directory
import re, os, errno, cgi, json, xml
import sys, codecs, locale, getopt
import xml.etree.ElementTree as ET
from subprocess import Popen, PIPE
#from BeautifulSoup import BeautifulStoneSoup
from operator import itemgetter
from xml.dom.minidom import parse, parseString
from os.path import expanduser
def HTMLEntitiesToUnicode(text):
"""Converts HTML entities to unicode. For example '&' becomes '&'."""
text = unicode(BeautifulStoneSoup(text, convertEntities=BeautifulStoneSoup.ALL_ENTITIES))
return text
def unicodeToHTMLEntities(text):
"""Converts unicode to HTML entities. For example '&' becomes '&'."""
text = cgi.escape(text).encode('ascii', 'xmlcharrefreplace')
return text
def indent(elem, level=0):
i = "\n" + level*" "
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
indent(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
def prettify(elem):
"""Return a pretty-printed XML string for the Element.
"""
rough_string = ElementTree.tostring(elem, 'utf-8')
reparsed = minidom.parseString(rough_string)
return reparsed.toprettyxml(indent="\t")
def getAMT(f,o_dir,src_only,wer):
"""Return a XML structure enriched with the Apertium MT output.
"""
print('... processing ' + str(f))
cwd = os.getcwd()
out_dir_path = os.path.join(cwd,o_dir)
if not os.path.exists(out_dir_path):
os.mkdir(out_dir_path)
o_root = ET.Element("html")
o_head = ET.SubElement(o_root, "head")
o_title = ET.SubElement(o_head, "title")
o_title.text = str(f)
o_style = ET.XML(table_style)
o_head.insert(2,o_style)
o_body = ET.SubElement(o_root, "body")
o_body.set("bgcolor", "#ffffff")
o_p1 = ET.SubElement(o_body, 'p')
o_p1.text = "File: " + str(f)
o_hline = ET.SubElement(o_body, 'hline')
o_table = ET.SubElement(o_body, "table")
o_table.set('class', 'tg')
i_tree = ET.parse(f)
i_root = i_tree.getroot()
i_tu_list = i_root.findall('.//tu')
for tu in i_tu_list:
tr_sme = ET.SubElement(o_table, 'tr')
th_sme = ET.SubElement(tr_sme, 'th')
th_sme.set('class', 'tg-sme')
th_sme.set('colspan', '2')
th_sme.text = tu[0][0].text
print('processing ' + str(tu[0][0].text) + '\n')
p = Popen('echo '+'\''+tu[0][0].text+'\''+cmd, shell=True, stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
if (not src_only):
tr_smn = ET.SubElement(o_table, 'tr')
th_smn = ET.SubElement(tr_smn, 'th')
th_smn.set('class', 'tg-smn')
th_smn.set('colspan', '2')
th_smn.text = tu[1][0].text
tr_amt = ET.SubElement(o_table, 'tr')
th_amt = ET.SubElement(tr_amt, 'th')
th_amt.set('class', 'tg-amt')
th_amt.set('colspan', '2')
th_amt.text = out.strip()
if (not wer):
th_amt.set('style', 'border-bottom: 2pt solid;')
if (wer):
tr_wer = ET.SubElement(o_table, 'tr')
td_wer = ET.SubElement(tr_wer, 'td')
td_wer.set('class', 'tg-wer')
td_wer.set('style', 'border-bottom: 5pt solid;')
td_wer.text = 'WER = value'
td_ble = ET.SubElement(tr_wer, 'td')
td_ble.set('class', 'tg-wer')
td_ble.set('style', 'border-bottom: 5pt solid;')
td_ble.text = 'BLEU = value'
file_name=os.path.basename(str(f))[:-3]+'html'
indent(o_root)
o_tree = ET.ElementTree(o_root)
o_tree.write(os.path.join(out_dir_path,str(file_name)),
xml_declaration=True,encoding='utf-8',
method="xml")
print('DONE ' + str(f) + '\n\n')
table_style = '<style type="text/css">\n.tg {border-collapse:collapse;border-spacing:0;}\n.tg td{font-family:Arial, sans-serif;font-size:14px;padding:8px 5px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;}\n.tg th{text-align:left;font-family:Arial, sans-serif;font-size:14px;font-weight:normal;padding:8px 5px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;}\n.tg .tg-sme{background-color:#c0c0c0;vertical-align:top;font-weight:bold;}\n.tg .tg-smn{background-color:#efefef;vertical-align:top;font-style:normal;}\n.tg .tg-amt{vertical-align:top;font-style:normal;}\n.tg .tg-wer{vertical-align:top;font-style:normal;color:grey;font-style: italic;}\n.hr.vertical{width: 0px; height: 100%;}</style>'
# parameters to be adjusted as needed
s_lang = 'sme'
t_lang = 'smn'
try:
os.environ["APERTIUM_HOME"]
except KeyError:
print("Please set the environment variable APERTIUM_HOME.\n" +
"This is the path to the directory where all apertium-SOURCE_LANG-TARGET_LANG reside.\n" +
"To set the variable you open the $HOME/.profile file and add the following two line:\n" +
"APERTIUM_HOME=/path/to/your/apertium/directory\n" +
"export APERTIUM_HOME\n" +
"The close the terminal and open it anew.")
sys.exit(1)
apertium_home=os.environ["APERTIUM_HOME"]
atm_dir = apertium_home + '/apertium' + '-' + s_lang + '-' + t_lang
#print("APT home is " + atm_dir)
#cmd = '| apertium -d ' + atm_dir + ' ' + s_lang + '-' + t_lang
# Change previous line to the following line if you want to see the hashform tags
cmd = "| apertium -d " + atm_dir + " " + s_lang + '-' + t_lang + '-dgen'
def main():
# parameters to be adjusted as needed
total = len(sys.argv)
i_file = ''
i_dir = 'tmx_data'
o_dir = 'otpt_dir'
src_only = True # True = no parallel, False = parallel
wer = False # True = WER, False = no WER
if (src_only):
wer = False
if (total == 3):
print('... total ' + str(total))
if str(sys.argv[1]) == '-f':
#print '... file ' + str(sys.argv[1])
i_file = str(sys.argv[2])
if i_file.endswith('tmx'):
getAMT(i_file,o_dir,src_only,wer)
if str(sys.argv[1]) == '-d':
i_dir = str(sys.argv[2])
for root, dirs, files in os.walk(i_dir): # Walk directory tree
print("Input dir {0} with {1} files ...".format(root, len(files)))
for f in files:
if f.endswith('tmx'):
getAMT(os.path.join(root,f),o_dir,src_only,wer)
else:
if (i_file == ''):
for root, dirs, files in os.walk(i_dir): # Walk directory tree
print("Input dir {0} with {1} files ...".format(root, len(files)))
for f in files:
if f.endswith('tmx'):
getAMT(os.path.join(i_dir,f),o_dir,src_only,wer)
if __name__ == "__main__":
reload(sys)
sys.setdefaultencoding("utf-8")
main()