| 22,6 → 22,7 |
|---|
| |
| import tsimapiak.dbconnector as dbconnector |
| import tsimapiak.parsenum as parsenum |
| import itertools |
| import re |
| |
| #wordlist = [{"id": 0, "navi": u"tawtute", "infix": u"tawtute", "type": u"n."}] + dbconnector.getnavilist() + [{"id": 0, "navi": u"na'vi", "infix": u"na'vi", "type": u"n."}] # XXX HACK - extra proper nouns |
| 31,22 → 32,22 |
|---|
| |
| # XXX HACK - These are words that are either not in Eana Eltu, or that get interpreted wrongly for whatever reason. The latter should be removed from this list when the parser gets more sophisticated. The former should also have an entry in the equivalent array in the translator! If it can take infixes, consider adding it to the main wordlist above (see the examples). The order is - original, Na'vi root, 0-pos infix, 1-pos infix, 2-pos infix, prefixes, suffixes. Things that can take affixes should go in the above list instead. |
| BROKENWORDS = ( |
| (u"sami", u"si", u"", u"am", u"", (()), (()), False), |
| (u"to", u"to", u"", u"", u"", (()), (()), False), |
| (u"sami", u"si", u"", u"am", u"", (()), (()), False, "si"), # otherwise parses as sa (tsa-lenited) + mi |
| #(u"to", u"to", u"", u"", u"", (()), (()), False), |
| #(u"frato", u"to", u"", u"", u"", [[u"fra"]], (()), False), |
| (u"soaiä", u"soaia", u"", u"", u"", (()), [[(u"ä", None)]], False), |
| (u"mengenga", u"ngenga", u"", u"", u"", [[u"me"]], (()), False), |
| (u"pxengenga", u"ngenga", u"", u"", u"", [[u"pxe"]], (()), False), |
| (u"kìmä", u"kä", u"", u"ìm", u"", (()), (()), False), |
| (u"apxay", u"pxay", u"", u"", u"", [[u"a"]], (()), False), |
| (u"soaiä", u"soaia", u"", u"", u"", (()), [[(u"ä", None)]], False, "soaia"), # does not parse, irregular form |
| #(u"mengenga", u"ngenga", u"", u"", u"", [[u"me"]], (()), False), |
| #(u"pxengenga", u"ngenga", u"", u"", u"", [[u"pxe"]], (()), False), |
| (u"kìmä", u"kä", u"", u"ìm", u"", (()), (()), False, "kä"), # otherwise parses as kìm (spin) + ä (genitive) |
| (u"apxay", u"pxay", u"", u"", u"", [[(u"a", "a")]], (()), False, "pxay"), # otherwise parses as apxa + -y (genitive) |
| #(u"akawng", u"kawng", u"", u"", u"", [[u"a"]], (()), False), |
| #(u"kawnga", u"kawng", u"", u"", u"", (()), [[(u"a", None)]], False), |
| #(u"kawng", u"kawng", u"", u"", u"", (()), (()), False), |
| (u"ka", u"ka", u"", u"", u"", (()), (()), False), |
| (u"uo", u"uo", u"", u"", u"", (()), (()), False), |
| (u"sìk", u"sìk", u"", u"", u"", (()), (()), False), |
| (u"sim", u"sim", u"", u"", u"", (()), (()), False), # probably not tsim lenited |
| (u"tse", u"tse", u"", u"", u"", (()), (()), False), |
| #(u"ka", u"ka", u"", u"", u"", (()), (()), False), |
| #(u"uo", u"uo", u"", u"", u"", (()), (()), False), |
| #(u"sìk", u"sìk", u"", u"", u"", (()), (()), False), |
| #(u"sim", u"sim", u"", u"", u"", (()), (()), False), # probably not tsim lenited |
| (u"tse", u"tse", u"", u"", u"", (()), (()), False, "tse"), # otherwise parses as tsa'u abbreviated (special case) |
| ) |
| |
| #INFIXES1 = (u"awn", u"eyk", u"us", u"äp", u"") |
| 59,12 → 60,12 |
|---|
| #prefixesv = ur"(?P<vpr>(?:nì|sä|tì|rä'ä |ke )?)" |
| |
| EXTRAINFIXES = [ |
| {"id": "-1", "navi": "eiy", "gloss": "LAUD.", "position": 2}, |
| {"id": "-2", "navi": "eng", "gloss": "PEJ.", "position": 2}, |
| {"id": "-1", "navi": "eiy", "orig_navi": "ei", "gloss": "LAUD.", "position": 2}, |
| {"id": "-2", "navi": "eng", "orig_navi": "äng", "gloss": "PEJ.", "position": 2}, |
| ] |
| |
| EXTRAPOSTFIXES = [ |
| {"id": "-3", "navi": "eyä", "gloss": "GEN."}, |
| {"id": "-3", "navi": "eyä", "orig_navi": "yä", "gloss": "GEN."}, |
| ] |
| |
| EXTRAADP = (("to", [x["id"] for x in wordlist if x["navi"] == "to"][0]), ("sì", [x["id"] for x in wordlist if x["navi"] == "sì"][0])) # words that act like adpositions but technically aren't |
| 78,7 → 79,7 |
|---|
| if prefix["navi"].startswith(letter): |
| new_prefix = prefix["navi"].replace(letter, replacement, 1) |
| if not [x for x in prefixes if x["navi"] == new_prefix]: # always assume a dictionary word over a lenited prefix |
| extraprefixes.append({"id": prefix["id"], "navi": new_prefix, "gloss": prefix["gloss"] + ".LENTD"}) |
| extraprefixes.append({"id": prefix["id"], "navi": new_prefix, "gloss": prefix["gloss"] + ".LENTD", "orig_navi": prefix["navi"]}) |
| break |
| |
| prefixes = sorted(prefixes + extraprefixes, key=lambda x: len(x["navi"]), reverse=True) |
| 85,6 → 86,40 |
|---|
| infixes = sorted(infixes + EXTRAINFIXES, key=lambda x: len(x["navi"]), reverse=True) |
| postfixes = sorted(postfixes + EXTRAPOSTFIXES, key=lambda x: len(x["navi"]), reverse=True) |
| |
| # Let's lenit the dictionary |
| extrawords = [] |
| for word in wordlist: |
| splitword = word["navi"].split(" ") |
| splitinfix = word["infix"].split(" ") |
| lenitword = {} |
| lenitinfix = {} |
| for i, wor in enumerate(splitword): |
| for letter, replacement in LENIT: |
| if wor.startswith(letter): |
| lenitword[i] = wor.replace(letter, replacement, 1) |
| lenitinfix[i] = splitinfix[i].replace(letter, replacement, 1) |
| break |
| |
| s = list(lenitword.keys()) |
| for lenits in itertools.chain.from_iterable(itertools.combinations(s, r) for r in range(1, len(s)+1)): |
| new_word = "" |
| new_infix = "" |
| for i, wor in enumerate(splitword): |
| if i in lenits: |
| new_word += lenitword[i] |
| new_infix += lenitinfix[i] |
| else: |
| new_word += wor |
| new_infix += splitinfix[i] |
| new_word += " " |
| new_infix += " " |
| print(f"Generated lenited {new_word} from {word['navi']}") |
| new_word = new_word[:-1] |
| new_infix = new_infix[:-1] |
| extrawords.append({"id": word["id"], "navi": new_word, "infix": new_infix, "type": word["type"], "lenited": True, "orig_navi": word["navi"]}) |
| |
| wordlist = sorted(wordlist + extrawords, key=lambda x: len(x["navi"]) * 2 + (0 if x["lenited"] else 1), reverse=True) |
| |
| def parseword(wordin): |
| tempid = 0 |
| temptype = u"" |
| 94,14 → 129,13 |
|---|
| if brokenword[1] == word["navi"]: |
| tempid = word["id"] |
| temptype = word["type"] |
| return {"word": {"id": tempid, "navi": brokenword[1], "infix": u"", "type": temptype}, "pref": brokenword[5], "post": brokenword[6], "len": brokenword[7], "inf": (brokenword[2], brokenword[3], brokenword[4]) } |
| ret = {"word": {"id": 0, "navi": u"[" + wordin[0] + u"]", "infix": u"", "type": u""}} |
| return {"word": {"id": tempid, "navi": brokenword[1], "infix": u"", "type": temptype, "orig_navi": brokenword[8]}, "pref": brokenword[5], "post": brokenword[6], "len": brokenword[7], "inf": (brokenword[2], brokenword[3], brokenword[4]) } |
| ret = {"word": {"id": 0, "navi": u"[" + wordin[0] + u"]", "infix": u"", "type": u"", "orig_navi": "[" + wordin[0] + "]"}, "len": False} |
| for word in wordlist: |
| word["navi"] = word["navi"].lower() |
| foundit = True |
| foundprefs = [] |
| foundposts = [] |
| lenited = False |
| splitword = word["infix"].split(u" ") |
| foundins = [u"", u"", u""] |
| if len(wordin) < len(splitword): |
| 140,14 → 174,6 |
|---|
| else: |
| if splitword[wor] in wordin[wor]: |
| center = splitword[wor] |
| if center == u"" and (wordin[wor] == u"paya" or splitword[wor] != u"pxay"): # XXX HACK - workaround to fix pay being lenited pxay. Maybe fixable without hardcoding? |
| for i in LENIT: |
| temp = u"" |
| if splitword[wor].startswith(i[0]): |
| temp = i[1] + splitword[wor][len(i[0]):] |
| if temp in wordin[wor]: |
| lenited = True |
| center = temp |
| if center == u"": |
| if splitword[wor].endswith(u"nga"): |
| temp = splitword[wor][:-3] + u"nge" |
| 196,13 → 222,13 |
|---|
| last = u"" |
| while last != pref: |
| last = pref |
| for pre in [x["navi"] for x in prefixes]: |
| for pre in prefixes: |
| if pref != u"": |
| if pref.endswith(pre): |
| if pre in foundprefs[wor]: |
| if pref.endswith(pre["navi"]): |
| if pre["navi"] in foundprefs[wor]: |
| break |
| foundprefs[wor].append(pre) |
| pref = pref[:-len(pre)] |
| foundprefs[wor].append((pre["navi"], pre["orig_navi"])) # only needed here, to handle lenition |
| pref = pref[:-len(pre["navi"])] |
| break |
| if pref != u"": |
| foundit = False |
| 216,7 → 242,7 |
|---|
| if posf.startswith(pos): |
| if (pos, posid) in foundposts[wor]: |
| break |
| if pos != u"ä" or word["navi"] != u"pey": # XXX HACK - fix for peyä. THIS SHOULD NOT BE HERE! |
| if pos != u"ä" or word["orig_navi"] != u"pey": # XXX HACK - fix for peyä. THIS SHOULD NOT BE HERE! |
| foundposts[wor].append((pos, posid)) |
| posf = posf[len(pos):] |
| break |
| 232,8 → 258,8 |
|---|
| ret["pref"] = foundprefs |
| ret["post"] = foundposts |
| ret["inf"] = foundins |
| ret["len"] = lenited |
| if foundit == True: |
| ret["len"] = word["lenited"] |
| ret["word"] = foundword |
| return ret |
| |