Rev 185 | Rev 187 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
56 | szabot | 1 | #!/usr/bin/python |
2 | # -*- coding: utf-8 -*- |
||
176 | muzer | 3 | # This file is part of Tsim Apiak. |
4 | # |
||
5 | # Tsim Apiak is free software: you can redistribute it and/or modify |
||
6 | # it under the terms of the GNU General Public Licence as published by |
||
7 | # the Free Software Foundation, either version 3 of the Licence, or |
||
8 | # (at your option) any later version. |
||
9 | # |
||
10 | # In addition to this, you must also comply with clause 4 of the |
||
11 | # Apache Licence, version 2.0, concerning attribution. Where there |
||
12 | # is a contradiction between the two licences, the GPL |
||
13 | # takes preference. |
||
14 | # |
||
186 | szabot | 15 | # Tsim Apiak is distributed in the hope that it will be useful, |
176 | muzer | 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
18 | # GNU General Public License for more details. |
||
19 | # |
||
20 | # You should have received a copy of the GNU General Public License |
||
21 | # along with Tsim Apiak. If not, see <http://www.gnu.org/licenses/>. |
||
56 | szabot | 22 | |
23 | import re |
||
66 | szabot | 24 | import dbconnector |
103 | szabot | 25 | import parsenum |
56 | szabot | 26 | |
65 | szabot | 27 | wordlist = dbconnector.getnavilist() |
28 | |||
94 | szabot | 29 | infixes1 = (u"awn", u"eyk", u"us", u"äp", u"") |
30 | infixes2 = (u"ìyev", u"iyev", u"ìmìy", u"arm", u"asy", u"ilv", u"ìmv", u"imv", u"ìrm", u"irv", u"ìsy", u"aly", u"ary", u"ìly", u"ìry", u"ìlm", u"alm", u"am", u"ay", u"er", u"ìm", u"iv", u"ìy", u"ol", u"") |
||
31 | infixes3 = (u"äng", u"ats", u"eiy", u"ei", u"uy", u"") |
||
156 | muzer | 32 | prefixes = (u"tsay", u"say", u"fay", u"fra", u"pxe", u"fne", u"tsa", u"sa", u"pe", u"fe", u"le", u"nì", u"sä", u"tì", u"sì", u"ay", u"me", u"fì", u"ke", u"he", u"a") |
171 | muzer | 33 | adpositions = (u"mungwrr", u"kxamlä", u"pximaw", u"pxisre", u"tafkip", u"nemfa", u"takip", u"mìkam", u"teri", u"fkip", u"luke", u"pxel", u"pxaw", u"rofa", u"ìlä", u"fpi", u"ftu", u"kip", u"lok", u"maw", u"sre", u"sìn", u"vay", u"eo", u"fa", u"hu", u"io", u"ka", u"mì", u"na", u"ne", u"ro", u"ta", u"uo", u"wä", u"äo", u"to", u"sì") |
121 | szabot | 34 | postfixes = adpositions + (u"tsyìp", u"eyä", u"ìri", u"ìl", u"it", u"lo", u"ri", u"ru", u"ti", u"ur", u"ve", u"yä", u"ya", u"tu", u"vi", u"yu", u"an", u"ng", u"ke", u"e", u"o", u"l", u"t", u"y", u"a", u"ä", u"r") |
62 | szabot | 35 | #prefixesn = ur"(?P<npr>(?:(?:fì|tsa)?(?:me|pxe|ay|fra)?|(?:fay)?|(?:tsay)?)(?:fne)?(?:tì|sä)?" |
74 | szabot | 36 | #prefixesv = ur"(?P<vpr>(?:nì|sä|tì|rä'ä |ke )?)" |
56 | szabot | 37 | |
91 | szabot | 38 | lenit = ((u"px", u"p"), (u"tx", u"t"), (u"kx", u"k"), (u"ts", u"s"), (u"t", u"s"), (u"p", u"f"), (u"k", u"h"), (u"'", u"")) |
39 | |||
56 | szabot | 40 | def parseword(wordin): |
90 | szabot | 41 | ret = {"word": {"id": 0, "navi": u"[" + wordin[0] + u"]", "infix": u"", "type": u""}} |
65 | szabot | 42 | for word in wordlist: |
43 | foundit = True |
||
44 | foundprefs = [] |
||
45 | foundposts = [] |
||
99 | szabot | 46 | lenited = False |
74 | szabot | 47 | splitword = word["infix"].split(u" ") |
172 | muzer | 48 | foundins = [u"", u"", u""] |
74 | szabot | 49 | if len(wordin) < len(splitword): |
68 | szabot | 50 | foundit = False |
51 | next |
||
65 | szabot | 52 | for wor in range(len(splitword)): |
76 | szabot | 53 | if not foundit: |
54 | break |
||
65 | szabot | 55 | foundprefs.append([]) |
56 | foundposts.append([]) |
||
57 | center = u"" |
||
58 | pre = [] |
||
59 | post = [] |
||
60 | if u"<1>" in splitword[wor]: |
||
185 | muzer | 61 | tempin1 = [] |
62 | tempin2 = [] |
||
63 | tempin3 = [] |
||
64 | for in1 in infixes1: |
||
65 | if in1 in wordin[wor]: |
||
66 | tempin1.append(in1) |
||
67 | for in2 in infixes2: |
||
68 | if in2 in wordin[wor]: |
||
69 | tempin2.append(in2) |
||
70 | for in3 in infixes3: |
||
71 | if in3 in wordin[wor]: |
||
72 | tempin3.append(in3) |
||
180 | szabot | 73 | for in1 in tempin1: |
74 | for in2 in tempin2: |
||
75 | for in3 in tempin3: |
||
65 | szabot | 76 | if splitword[wor].replace(u"<1><2>",in1+in2).replace(u"<3>",in3) in wordin[wor]: |
77 | center = splitword[wor].replace(u"<1><2>",in1+in2).replace(u"<3>",in3) |
||
78 | foundins = [in1, in2, in3] |
||
79 | break |
||
75 | szabot | 80 | if center != u"": break |
81 | if center != u"": break |
||
65 | szabot | 82 | else: |
83 | if splitword[wor] in wordin[wor]: |
||
84 | center = splitword[wor] |
||
95 | szabot | 85 | if center == u"": |
91 | szabot | 86 | for i in lenit: |
92 | szabot | 87 | temp = u"" |
91 | szabot | 88 | if splitword[wor].startswith(i[0]): |
92 | szabot | 89 | temp = i[1] + splitword[wor][len(i[0]):] |
90 | if temp in wordin[wor]: |
||
99 | szabot | 91 | lenited = True |
92 | szabot | 92 | center = temp |
95 | szabot | 93 | if center == u"": |
94 | if splitword[wor].endswith(u"nga"): |
||
97 | szabot | 95 | temp = splitword[wor][:-3] + u"ng" |
95 | szabot | 96 | if temp in wordin[wor]: |
97 | center = temp |
||
103 | szabot | 98 | if splitword[wor].endswith(u"po"): |
174 | muzer | 99 | temp = splitword[wor][:-2] + u"p" |
103 | szabot | 100 | if temp in wordin[wor]: |
101 | center = temp |
||
74 | szabot | 102 | if center == u"": |
65 | szabot | 103 | foundit = False |
104 | break |
||
91 | szabot | 105 | temp = wordin[wor].split(center) |
106 | if len(temp) != 2: |
||
107 | foundit = False |
||
108 | break |
||
109 | pref, posf = temp |
||
119 | szabot | 110 | last = u"" |
111 | while last != pref: |
||
112 | last = pref |
||
113 | for pre in prefixes: |
||
114 | if pref != u"": |
||
115 | if pref.endswith(pre): |
||
116 | if pre in foundprefs[wor]: |
||
117 | break |
||
118 | foundprefs[wor].append(pre) |
||
119 | pref = pref[:-len(pre)] |
||
120 | szabot | 120 | break |
65 | szabot | 121 | if pref != u"": |
122 | foundit = False |
||
123 | break |
||
119 | szabot | 124 | last = u"" |
125 | while last != posf: |
||
126 | last = posf |
||
127 | for pos in postfixes: |
||
128 | if posf != u"": |
||
129 | if posf.startswith(pos): |
||
130 | if pos in foundposts[wor]: |
||
131 | break |
||
132 | foundposts[wor].append(pos) |
||
133 | posf = posf[len(pos):] |
||
120 | szabot | 134 | break |
82 | szabot | 135 | if posf != u"": |
80 | szabot | 136 | foundit = False |
137 | break |
||
65 | szabot | 138 | if foundit == True: |
139 | foundword = word |
||
56 | szabot | 140 | break |
87 | szabot | 141 | ret["pref"] = foundprefs |
142 | ret["post"] = foundposts |
||
143 | ret["inf"] = foundins |
||
99 | szabot | 144 | ret["len"] = lenited |
65 | szabot | 145 | if foundit == True: |
71 | szabot | 146 | ret["word"] = foundword |
77 | szabot | 147 | return ret |
148 | |||
149 | def parsesent(sent): |
||
101 | szabot | 150 | sent = sent.strip().lower().replace(u"’", u"'") |
100 | szabot | 151 | sent = re.sub(ur"[^\wìä' ]",u"",sent) |
89 | szabot | 152 | sent = re.sub(ur"\ +",u" ",sent) |
153 | sent = sent.split(u" ") |
||
77 | szabot | 154 | ret = [] |
155 | left = len(sent) |
||
156 | while left: |
||
108 | szabot | 157 | word = parsenum.parse(sent[len(sent)-left]) |
103 | szabot | 158 | if word == None: |
159 | word = parseword(sent[-left:]) |
||
78 | szabot | 160 | left -= len(word["word"]["navi"].split(" ")) |
77 | szabot | 161 | ret.append(word) |
136 | muzer | 162 | return ret |