Download or view Unicode.frink in plain text format
/* This program contains routines to handle Unicode, especially those functions
requiring downloading of Unicode tables to work correctly.
Frink already has lots of powerful Unicode-aware string functions, which can
be seen in:
https://frinklang.org/#CorrectStringParsing
Much other Unicode testing of single characters can be done through the
Java class Character. For example:
callJava["java.lang.Character", "isWhitespace", char["\t"]]
which returns true.
The java.lang.Character class is disastrous and poorly-designed, though, with
classes like getType not returning consistent bitmapped values.
*/
class Unicode
{
/** A dictionary mapping from codepoint (as integer) -> codepoint name */
class var codepointNames = undef
/** A dictionary mapping from char->chars of confusable characters. */
class var confusablesDict = undef
/** A flag indicating if the "confusables" dictionary has been loaded. */
class var confusablesLoaded = false
/** The name of the Java class that knows characters. */
class var CHARCLASS = "java.lang.Character";
/** This returns the Unicode codepoint name for a codepoint (specified as an
integer or a string containing multiple codepoints) */
class getCodepointName[i] :=
{
if codepointNames == undef
loadCodepointNames[]
if isInteger[i]
return codepointNames@i
else
if isString[i]
{
result = new array
for c = chars[i]
result.push[codepointNames@c]
return result
} else
{
result = new array
for c = i
result.push[getCodepointName[c]]
return result
}
}
/** This searches the codepoint names for values that match a specific
regular expression. */
class searchNames[pattern] :=
{
if codepointNames == undef
loadCodepointNames[]
retval = new array
for [codepoint, name] = codepointNames
if name =~ pattern
retval.push[[codepoint, char[codepoint], name]]
// Sort by codepoint
return sort[retval, byColumn[0]]
}
/** This is a human-readable name search. It takes a regular expression
and returns names that match that pattern. */
class prettySearchNames[pattern] :=
{
ret = ""
for [dec, char, name] = Unicode.searchNames[pattern]
ret = ret + toASCIIHigh[char[dec]] + "\t" + char + "\t" + name + "\n"
return ret
}
/** This is a private method that loads the dictionary of codepoint names.
*/
class loadCodepointNames[] :=
{
codepointNames = new dict
min = staticJava[CHARCLASS, "MIN_CODE_POINT"]
// max = staticJava[CHARCLASS, "MAX_CODE_POINT"]
max = 0x1F9FF
for i = min to max
if callJava[CHARCLASS, "isDefined", i]
codepointNames@i = charName[i]
}
/** deconfuse[string]: This follows the procedure in Unicode Technical
Standard #39 for deconfusing similar characters in a string.
http://www.unicode.org/reports/tr39/
Specifically, section 4, "Confusable Detection".
It uses the "confusables" table available at:
http://www.unicode.org/Public/security/latest/confusables.txt
to perform the deconfusing.
You generally don't want to call this with a single string, but to call
deconfuse[x] == deconfuse[y]
to see if two separate strings are confusable. Or, if
deconfuse[x] != x
then you might have reason to believe that the string x had confusable
characters in it, and you might want to treat it with stronger security.
It's *still* a rather weak notion of confusable, as combining characters
and accents are not folded together. You may want to first use
normalizeUnicode[str] to perform some of this normalization.
You might want to use something stronger like folding Unicode to ASCII:
https://github.com/ericxtang/sunspot/blob/deafdd55f2a9534cc96471958ea1c206430832e7/sunspot/solr/solr/conf/mapping-FoldToASCII.txt
*/
class deconfuse[str] :=
{
nfd = normalizeUnicode[str, "NFD"]
loadConfusables[]
result = new array
for c = charList[nfd]
{
if confusablesDict.containsKey[c]
result.push[confusablesDict@c]
else
result.push[c]
}
return normalizeUnicode[join["", result], "NFD"]
}
/** Internal function to load the Unicode "confusables" file. */
class loadConfusables[] :=
{
if confusablesLoaded
return
// This is a dictionary from source char to target string.
confusablesDict = new dict
// TODO: Cache this file somewhere
for line = lines["http://www.unicode.org/Public/security/latest/confusables.txt"]
{
if line =~ %r/^\s*#/
next
if [source, target] = line =~ %r/([A-F0-9]{4,6})\s*;\s*([\sA-F0-9]+)/
{
target = trim[target]
sourceStr = char[parseInt[source, 16]]
targetStr = char[map[{|x| parseInt[x,16]},
split[" ", trim[target]]]]
confusablesDict@sourceStr = targetStr
// println["$sourceStr\t$targetStr"]
}
}
confusablesLoaded = true
}
}
/*
dumpChars[x] := println["$x\t" + uc[hex[char[x]]]]
original = "Inglês"
original = "\u2487" // This is a Unicode character indicating parenthesized 20
dumpChars[original]
deconfuse = Unicode.deconfuse[original]
dumpChars[deconfuse]
dumpChars[Unicode.deconfuse["(2O)"]] // This is a letter capital O, not a zero
*/
Download or view Unicode.frink in plain text format
This is a program written in the programming language Frink.
For more information, view the Frink
Documentation or see More Sample Frink Programs.
Alan Eliasen was born 19966 days, 17 hours, 40 minutes ago.