blob: 3131f5687c6a7fa14a364b89f4f6fab45078ea12 [file] [log] [blame]
Roozbeh Pournader0e969e22016-03-09 23:08:45 -08001#!/usr/bin/env python
2
3import collections
Roozbeh Pournader5dde0872016-03-31 13:54:56 -07004import copy
Roozbeh Pournader0e969e22016-03-09 23:08:45 -08005import glob
6from os import path
Seigo Nonaka56811242021-04-16 00:11:43 -07007import re
Roozbeh Pournader0e969e22016-03-09 23:08:45 -08008import sys
9from xml.etree import ElementTree
10
11from fontTools import ttLib
12
Roozbeh Pournader5dde0872016-03-31 13:54:56 -070013EMOJI_VS = 0xFE0F
14
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080015LANG_TO_SCRIPT = {
Calvin Pan24479da2021-12-14 18:50:31 +080016 'af': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070017 'as': 'Beng',
Calvin Panfb114cc2021-12-06 21:42:12 +080018 'am': 'Latn',
Roozbeh Pournader7e04dd12017-10-13 17:41:31 -070019 'be': 'Cyrl',
Roozbeh Pournader033b2222017-02-22 18:53:39 -080020 'bg': 'Cyrl',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070021 'bn': 'Beng',
Calvin Pan24479da2021-12-14 18:50:31 +080022 'cs': 'Latn',
Roozbeh Pournader033b2222017-02-22 18:53:39 -080023 'cu': 'Cyrl',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070024 'cy': 'Latn',
25 'da': 'Latn',
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080026 'de': 'Latn',
Calvin Pan24479da2021-12-14 18:50:31 +080027 'el': 'Latn',
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080028 'en': 'Latn',
29 'es': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070030 'et': 'Latn',
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080031 'eu': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070032 'fr': 'Latn',
33 'ga': 'Latn',
Calvin Panfb114cc2021-12-06 21:42:12 +080034 'gl': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070035 'gu': 'Gujr',
36 'hi': 'Deva',
37 'hr': 'Latn',
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080038 'hu': 'Latn',
39 'hy': 'Armn',
Calvin Panfb114cc2021-12-06 21:42:12 +080040 'it': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070041 'ja': 'Jpan',
Calvin Pan24479da2021-12-14 18:50:31 +080042 'ka': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070043 'kn': 'Knda',
44 'ko': 'Kore',
Roozbeh Pournader7e04dd12017-10-13 17:41:31 -070045 'la': 'Latn',
Calvin Panfb114cc2021-12-06 21:42:12 +080046 'lt': 'Latn',
Calvin Pan24479da2021-12-14 18:50:31 +080047 'lv': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070048 'ml': 'Mlym',
49 'mn': 'Cyrl',
50 'mr': 'Deva',
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080051 'nb': 'Latn',
Calvin Pan24479da2021-12-14 18:50:31 +080052 'nl': 'Latn',
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080053 'nn': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070054 'or': 'Orya',
55 'pa': 'Guru',
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080056 'pt': 'Latn',
Calvin Pan24479da2021-12-14 18:50:31 +080057 'sk': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070058 'sl': 'Latn',
Calvin Pan24479da2021-12-14 18:50:31 +080059 'sq': 'Latn',
Calvin Pan1e966a32022-01-12 16:12:38 +080060 'sv': 'Latn',
Jungshik Shin6c4f9e02016-03-19 09:32:34 -070061 'ta': 'Taml',
62 'te': 'Telu',
63 'tk': 'Latn',
Calvin Panfb114cc2021-12-06 21:42:12 +080064 'uk': 'Latn',
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080065}
66
67def lang_to_script(lang_code):
68 lang = lang_code.lower()
69 while lang not in LANG_TO_SCRIPT:
70 hyphen_idx = lang.rfind('-')
71 assert hyphen_idx != -1, (
72 'We do not know what script the "%s" language is written in.'
73 % lang_code)
74 assumed_script = lang[hyphen_idx+1:]
75 if len(assumed_script) == 4 and assumed_script.isalpha():
76 # This is actually the script
77 return assumed_script.title()
78 lang = lang[:hyphen_idx]
79 return LANG_TO_SCRIPT[lang]
80
81
Roozbeh Pournader5dde0872016-03-31 13:54:56 -070082def printable(inp):
83 if type(inp) is set: # set of character sequences
84 return '{' + ', '.join([printable(seq) for seq in inp]) + '}'
85 if type(inp) is tuple: # character sequence
86 return '<' + (', '.join([printable(ch) for ch in inp])) + '>'
87 else: # single character
88 return 'U+%04X' % inp
89
90
91def open_font(font):
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080092 font_file, index = font
93 font_path = path.join(_fonts_dir, font_file)
94 if index is not None:
Roozbeh Pournader5dde0872016-03-31 13:54:56 -070095 return ttLib.TTFont(font_path, fontNumber=index)
Roozbeh Pournader0e969e22016-03-09 23:08:45 -080096 else:
Roozbeh Pournader5dde0872016-03-31 13:54:56 -070097 return ttLib.TTFont(font_path)
98
99
100def get_best_cmap(font):
101 ttfont = open_font(font)
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800102 all_unicode_cmap = None
103 bmp_cmap = None
104 for cmap in ttfont['cmap'].tables:
105 specifier = (cmap.format, cmap.platformID, cmap.platEncID)
106 if specifier == (4, 3, 1):
107 assert bmp_cmap is None, 'More than one BMP cmap in %s' % (font, )
108 bmp_cmap = cmap
109 elif specifier == (12, 3, 10):
110 assert all_unicode_cmap is None, (
111 'More than one UCS-4 cmap in %s' % (font, ))
112 all_unicode_cmap = cmap
113
114 return all_unicode_cmap.cmap if all_unicode_cmap else bmp_cmap.cmap
115
116
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700117def get_variation_sequences_cmap(font):
118 ttfont = open_font(font)
119 vs_cmap = None
120 for cmap in ttfont['cmap'].tables:
121 specifier = (cmap.format, cmap.platformID, cmap.platEncID)
122 if specifier == (14, 0, 5):
123 assert vs_cmap is None, 'More than one VS cmap in %s' % (font, )
124 vs_cmap = cmap
125 return vs_cmap
126
127
128def get_emoji_map(font):
129 # Add normal characters
130 emoji_map = copy.copy(get_best_cmap(font))
Seigo Nonaka2e9e7922021-05-04 17:12:35 -0700131 reverse_cmap = {glyph: code for code, glyph in emoji_map.items() if not contains_pua(code) }
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700132
133 # Add variation sequences
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700134 vs_cmap = get_variation_sequences_cmap(font)
135 if vs_cmap:
136 for vs in vs_cmap.uvsDict:
137 for base, glyph in vs_cmap.uvsDict[vs]:
138 if glyph is None:
139 emoji_map[(base, vs)] = emoji_map[base]
140 else:
141 emoji_map[(base, vs)] = glyph
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700142
143 # Add GSUB rules
144 ttfont = open_font(font)
145 for lookup in ttfont['GSUB'].table.LookupList.Lookup:
Roozbeh Pournaderaa3ee8e2017-04-10 13:52:20 -0700146 if lookup.LookupType != 4:
147 # Other lookups are used in the emoji font for fallback.
148 # We ignore them for now.
149 continue
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700150 for subtable in lookup.SubTable:
151 ligatures = subtable.ligatures
152 for first_glyph in ligatures:
153 for ligature in ligatures[first_glyph]:
154 sequence = [first_glyph] + ligature.Component
155 sequence = [reverse_cmap[glyph] for glyph in sequence]
156 sequence = tuple(sequence)
157 # Make sure no starting subsequence of 'sequence' has been
158 # seen before.
159 for sub_len in range(2, len(sequence)+1):
160 subsequence = sequence[:sub_len]
161 assert subsequence not in emoji_map
162 emoji_map[sequence] = ligature.LigGlyph
163
164 return emoji_map
165
166
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800167def assert_font_supports_any_of_chars(font, chars):
168 best_cmap = get_best_cmap(font)
169 for char in chars:
170 if char in best_cmap:
171 return
172 sys.exit('None of characters in %s were found in %s' % (chars, font))
173
174
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700175def assert_font_supports_all_of_chars(font, chars):
176 best_cmap = get_best_cmap(font)
177 for char in chars:
178 assert char in best_cmap, (
179 'U+%04X was not found in %s' % (char, font))
180
181
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700182def assert_font_supports_none_of_chars(font, chars, fallbackName):
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700183 best_cmap = get_best_cmap(font)
184 for char in chars:
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700185 if fallbackName:
186 assert char not in best_cmap, 'U+%04X was found in %s' % (char, font)
187 else:
188 assert char not in best_cmap, (
189 'U+%04X was found in %s in fallback %s' % (char, font, fallbackName))
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700190
191
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700192def assert_font_supports_all_sequences(font, sequences):
193 vs_dict = get_variation_sequences_cmap(font).uvsDict
194 for base, vs in sorted(sequences):
195 assert vs in vs_dict and (base, None) in vs_dict[vs], (
196 '<U+%04X, U+%04X> was not found in %s' % (base, vs, font))
197
198
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800199def check_hyphens(hyphens_dir):
200 # Find all the scripts that need automatic hyphenation
201 scripts = set()
202 for hyb_file in glob.iglob(path.join(hyphens_dir, '*.hyb')):
203 hyb_file = path.basename(hyb_file)
204 assert hyb_file.startswith('hyph-'), (
205 'Unknown hyphenation file %s' % hyb_file)
206 lang_code = hyb_file[hyb_file.index('-')+1:hyb_file.index('.')]
207 scripts.add(lang_to_script(lang_code))
208
209 HYPHENS = {0x002D, 0x2010}
210 for script in scripts:
211 fonts = _script_to_font_map[script]
212 assert fonts, 'No fonts found for the "%s" script' % script
213 for font in fonts:
214 assert_font_supports_any_of_chars(font, HYPHENS)
215
216
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700217class FontRecord(object):
Seigo Nonaka56811242021-04-16 00:11:43 -0700218 def __init__(self, name, psName, scripts, variant, weight, style, fallback_for, font):
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700219 self.name = name
Seigo Nonaka56811242021-04-16 00:11:43 -0700220 self.psName = psName
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700221 self.scripts = scripts
222 self.variant = variant
223 self.weight = weight
224 self.style = style
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700225 self.fallback_for = fallback_for
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700226 self.font = font
227
228
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800229def parse_fonts_xml(fonts_xml_path):
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700230 global _script_to_font_map, _fallback_chains, _all_fonts
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800231 _script_to_font_map = collections.defaultdict(set)
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700232 _fallback_chains = {}
233 _all_fonts = []
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800234 tree = ElementTree.parse(fonts_xml_path)
Seigo Nonaka9092dc22017-01-06 16:54:52 +0900235 families = tree.findall('family')
236 # Minikin supports up to 254 but users can place their own font at the first
237 # place. Thus, 253 is the maximum allowed number of font families in the
238 # default collection.
239 assert len(families) < 254, (
240 'System font collection can contains up to 253 font families.')
241 for family in families:
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800242 name = family.get('name')
243 variant = family.get('variant')
244 langs = family.get('lang')
245 if name:
246 assert variant is None, (
247 'No variant expected for LGC font %s.' % name)
248 assert langs is None, (
249 'No language expected for LGC fonts %s.' % name)
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700250 assert name not in _fallback_chains, 'Duplicated name entry %s' % name
251 _fallback_chains[name] = []
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800252 else:
253 assert variant in {None, 'elegant', 'compact'}, (
254 'Unexpected value for variant: %s' % variant)
255
Seigo Nonaka56811242021-04-16 00:11:43 -0700256 trim_re = re.compile(r"^[ \n\r\t]*(.+)[ \n\r\t]*$")
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700257 for family in families:
258 name = family.get('name')
259 variant = family.get('variant')
260 langs = family.get('lang')
261
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800262 if langs:
263 langs = langs.split()
264 scripts = {lang_to_script(lang) for lang in langs}
265 else:
266 scripts = set()
267
268 for child in family:
269 assert child.tag == 'font', (
270 'Unknown tag <%s>' % child.tag)
Jungshik Shin88b11142017-03-17 14:56:17 -0700271 font_file = child.text.rstrip()
Seigo Nonaka56811242021-04-16 00:11:43 -0700272
273 m = trim_re.match(font_file)
274 font_file = m.group(1)
275
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800276 weight = int(child.get('weight'))
277 assert weight % 100 == 0, (
278 'Font weight "%d" is not a multiple of 100.' % weight)
279
280 style = child.get('style')
281 assert style in {'normal', 'italic'}, (
282 'Unknown style "%s"' % style)
283
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700284 fallback_for = child.get('fallbackFor')
285
286 assert not name or not fallback_for, (
287 'name and fallbackFor cannot be present at the same time')
288 assert not fallback_for or fallback_for in _fallback_chains, (
289 'Unknown fallback name: %s' % fallback_for)
290
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800291 index = child.get('index')
292 if index:
293 index = int(index)
294
Seigo Nonaka56811242021-04-16 00:11:43 -0700295 if not path.exists(path.join(_fonts_dir, m.group(1))):
Seigo Nonaka1403ff22018-01-18 17:24:31 -0800296 continue # Missing font is a valid case. Just ignore the missing font files.
297
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700298 record = FontRecord(
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800299 name,
Seigo Nonaka56811242021-04-16 00:11:43 -0700300 child.get('postScriptName'),
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800301 frozenset(scripts),
302 variant,
303 weight,
304 style,
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700305 fallback_for,
306 (font_file, index))
307
308 _all_fonts.append(record)
309
310 if not fallback_for:
311 if not name or name == 'sans-serif':
Haibo Huang715857d2020-03-05 11:58:47 -0800312 for _, fallback in _fallback_chains.items():
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700313 fallback.append(record)
314 else:
315 _fallback_chains[name].append(record)
316 else:
317 _fallback_chains[fallback_for].append(record)
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800318
319 if name: # non-empty names are used for default LGC fonts
320 map_scripts = {'Latn', 'Grek', 'Cyrl'}
321 else:
322 map_scripts = scripts
323 for script in map_scripts:
324 _script_to_font_map[script].add((font_file, index))
325
326
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700327def check_emoji_coverage(all_emoji, equivalent_emoji):
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700328 emoji_fonts = get_emoji_fonts()
329 check_emoji_font_coverage(emoji_fonts, all_emoji, equivalent_emoji)
Doug Feltf874a192016-07-08 17:42:15 -0700330
331
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700332def get_emoji_fonts():
333 return [ record.font for record in _all_fonts if 'Zsye' in record.scripts ]
Doug Feltf874a192016-07-08 17:42:15 -0700334
Seigo Nonaka2e9e7922021-05-04 17:12:35 -0700335def is_pua(x):
336 return 0xE000 <= x <= 0xF8FF or 0xF0000 <= x <= 0xFFFFD or 0x100000 <= x <= 0x10FFFD
337
338def contains_pua(sequence):
339 if type(sequence) is tuple:
340 return any([is_pua(x) for x in sequence])
341 else:
342 return is_pua(sequence)
343
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700344def get_psname(ttf):
345 return str(next(x for x in ttf['name'].names
346 if x.platformID == 3 and x.platEncID == 1 and x.nameID == 6))
Seigo Nonaka2e9e7922021-05-04 17:12:35 -0700347
348def check_emoji_compat():
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700349 for emoji_font in get_emoji_fonts():
350 ttf = open_font(emoji_font)
351 psname = get_psname(ttf)
Seigo Nonaka2e9e7922021-05-04 17:12:35 -0700352
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700353 # If the font file is NotoColorEmoji, it must be Compat font.
354 if psname == 'NotoColorEmoji':
355 meta = ttf['meta']
356 assert meta, 'Compat font must have meta table'
357 assert 'Emji' in meta.data, 'meta table should have \'Emji\' data.'
358
359def check_emoji_font_coverage(emoji_fonts, all_emoji, equivalent_emoji):
360 coverages = []
361 for emoji_font in emoji_fonts:
362 coverages.append(get_emoji_map(emoji_font))
Rod Se34a19d2020-03-16 00:01:15 -0700363
364 errors = []
365
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700366 for sequence in all_emoji:
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700367 if all([sequence not in coverage for coverage in coverages]):
368 errors.append('%s is not supported in the emoji font.' % printable(sequence))
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700369
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700370 for coverage in coverages:
371 for sequence in coverage:
372 if sequence in {0x0000, 0x000D, 0x0020}:
373 # The font needs to support a few extra characters, which is OK
374 continue
Seigo Nonaka2e9e7922021-05-04 17:12:35 -0700375
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700376 if contains_pua(sequence):
377 # The font needs to have some PUA for EmojiCompat library.
378 continue
Seigo Nonaka2e9e7922021-05-04 17:12:35 -0700379
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700380 if sequence not in all_emoji:
381 errors.append('%s support unexpected in the emoji font.' % printable(sequence))
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700382
Haibo Huang715857d2020-03-05 11:58:47 -0800383 for first, second in equivalent_emoji.items():
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700384 for coverage in coverages:
385 if first not in coverage or second not in coverage:
386 continue # sequence will be reported missing
387 if coverage[first] != coverage[second]:
388 errors.append('%s and %s should map to the same glyph.' % (
389 printable(first),
390 printable(second)))
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700391
Seigo Nonakaebdae2e2021-06-14 17:38:22 -0700392 for coverage in coverages:
393 for glyph in set(coverage.values()):
394 maps_to_glyph = [
395 seq for seq in coverage if coverage[seq] == glyph and not contains_pua(seq) ]
396 if len(maps_to_glyph) > 1:
397 # There are more than one sequences mapping to the same glyph. We
398 # need to make sure they were expected to be equivalent.
399 equivalent_seqs = set()
400 for seq in maps_to_glyph:
401 equivalent_seq = seq
402 while equivalent_seq in equivalent_emoji:
403 equivalent_seq = equivalent_emoji[equivalent_seq]
404 equivalent_seqs.add(equivalent_seq)
405 if len(equivalent_seqs) != 1:
406 errors.append('The sequences %s should not result in the same glyph %s' % (
407 printable(equivalent_seqs),
408 glyph))
Roozbeh Pournader3b3c78e2016-07-25 14:04:34 -0700409
Rod Se34a19d2020-03-16 00:01:15 -0700410 assert not errors, '%d emoji font errors:\n%s\n%d emoji font coverage errors' % (len(errors), '\n'.join(errors), len(errors))
411
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700412
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700413def check_emoji_defaults(default_emoji):
414 missing_text_chars = _emoji_properties['Emoji'] - default_emoji
Haibo Huang715857d2020-03-05 11:58:47 -0800415 for name, fallback_chain in _fallback_chains.items():
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700416 emoji_font_seen = False
417 for record in fallback_chain:
418 if 'Zsye' in record.scripts:
419 emoji_font_seen = True
420 # No need to check the emoji font
421 continue
422 # For later fonts, we only check them if they have a script
423 # defined, since the defined script may get them to a higher
424 # score even if they appear after the emoji font. However,
425 # we should skip checking the text symbols font, since
426 # symbol fonts should be able to override the emoji display
427 # style when 'Zsym' is explicitly specified by the user.
428 if emoji_font_seen and (not record.scripts or 'Zsym' in record.scripts):
429 continue
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700430
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700431 # Check default emoji-style characters
Haibo Huang715857d2020-03-05 11:58:47 -0800432 assert_font_supports_none_of_chars(record.font, default_emoji, name)
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700433
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700434 # Mark default text-style characters appearing in fonts above the emoji
435 # font as seen
436 if not emoji_font_seen:
437 missing_text_chars -= set(get_best_cmap(record.font))
Roozbeh Pournader7b822e52016-03-16 18:55:32 -0700438
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700439 # Noto does not have monochrome glyphs for Unicode 7.0 wingdings and
440 # webdings yet.
441 missing_text_chars -= _chars_by_age['7.0']
442 assert missing_text_chars == set(), (
443 'Text style version of some emoji characters are missing: ' +
444 repr(missing_text_chars))
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700445
446
Roozbeh Pournader7b822e52016-03-16 18:55:32 -0700447# Setting reverse to true returns a dictionary that maps the values to sets of
448# characters, useful for some binary properties. Otherwise, we get a
449# dictionary that maps characters to the property values, assuming there's only
450# one property in the file.
451def parse_unicode_datafile(file_path, reverse=False):
452 if reverse:
453 output_dict = collections.defaultdict(set)
454 else:
455 output_dict = {}
456 with open(file_path) as datafile:
457 for line in datafile:
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700458 if '#' in line:
459 line = line[:line.index('#')]
460 line = line.strip()
461 if not line:
462 continue
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700463
Roozbeh Pournader3b3c78e2016-07-25 14:04:34 -0700464 chars, prop = line.split(';')[:2]
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700465 chars = chars.strip()
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700466 prop = prop.strip()
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700467
468 if ' ' in chars: # character sequence
469 sequence = [int(ch, 16) for ch in chars.split(' ')]
470 additions = [tuple(sequence)]
471 elif '..' in chars: # character range
472 char_start, char_end = chars.split('..')
473 char_start = int(char_start, 16)
474 char_end = int(char_end, 16)
Haibo Huang715857d2020-03-05 11:58:47 -0800475 additions = range(char_start, char_end+1)
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700476 else: # singe character
477 additions = [int(chars, 16)]
Roozbeh Pournader7b822e52016-03-16 18:55:32 -0700478 if reverse:
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700479 output_dict[prop].update(additions)
Roozbeh Pournader7b822e52016-03-16 18:55:32 -0700480 else:
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700481 for addition in additions:
482 assert addition not in output_dict
483 output_dict[addition] = prop
Roozbeh Pournader7b822e52016-03-16 18:55:32 -0700484 return output_dict
485
486
Roozbeh Pournaderaa3ee8e2017-04-10 13:52:20 -0700487def parse_emoji_variants(file_path):
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700488 emoji_set = set()
489 text_set = set()
490 with open(file_path) as datafile:
491 for line in datafile:
492 if '#' in line:
493 line = line[:line.index('#')]
494 line = line.strip()
495 if not line:
496 continue
497 sequence, description, _ = line.split(';')
498 sequence = sequence.strip().split(' ')
499 base = int(sequence[0], 16)
500 vs = int(sequence[1], 16)
501 description = description.strip()
502 if description == 'text style':
503 text_set.add((base, vs))
504 elif description == 'emoji style':
505 emoji_set.add((base, vs))
506 return text_set, emoji_set
507
508
Roozbeh Pournader7b822e52016-03-16 18:55:32 -0700509def parse_ucd(ucd_path):
510 global _emoji_properties, _chars_by_age
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700511 global _text_variation_sequences, _emoji_variation_sequences
512 global _emoji_sequences, _emoji_zwj_sequences
Roozbeh Pournader7b822e52016-03-16 18:55:32 -0700513 _emoji_properties = parse_unicode_datafile(
514 path.join(ucd_path, 'emoji-data.txt'), reverse=True)
Roozbeh Pournaderf7a68c12017-04-04 18:59:31 -0700515 emoji_properties_additions = parse_unicode_datafile(
516 path.join(ucd_path, 'additions', 'emoji-data.txt'), reverse=True)
517 for prop in emoji_properties_additions.keys():
518 _emoji_properties[prop].update(emoji_properties_additions[prop])
519
Roozbeh Pournader7b822e52016-03-16 18:55:32 -0700520 _chars_by_age = parse_unicode_datafile(
521 path.join(ucd_path, 'DerivedAge.txt'), reverse=True)
Roozbeh Pournaderaa3ee8e2017-04-10 13:52:20 -0700522 sequences = parse_emoji_variants(
523 path.join(ucd_path, 'emoji-variation-sequences.txt'))
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700524 _text_variation_sequences, _emoji_variation_sequences = sequences
525 _emoji_sequences = parse_unicode_datafile(
526 path.join(ucd_path, 'emoji-sequences.txt'))
Siyamed Sinir6e06ad02017-04-19 18:18:35 -0700527 _emoji_sequences.update(parse_unicode_datafile(
528 path.join(ucd_path, 'additions', 'emoji-sequences.txt')))
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700529 _emoji_zwj_sequences = parse_unicode_datafile(
530 path.join(ucd_path, 'emoji-zwj-sequences.txt'))
Roozbeh Pournader1800ba42017-03-17 18:23:23 -0700531 _emoji_zwj_sequences.update(parse_unicode_datafile(
532 path.join(ucd_path, 'additions', 'emoji-zwj-sequences.txt')))
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700533
Siyamed Sinird97df5a2018-04-12 13:11:42 -0700534 exclusions = parse_unicode_datafile(path.join(ucd_path, 'additions', 'emoji-exclusions.txt'))
535 _emoji_sequences = remove_emoji_exclude(_emoji_sequences, exclusions)
536 _emoji_zwj_sequences = remove_emoji_exclude(_emoji_zwj_sequences, exclusions)
537 _emoji_variation_sequences = remove_emoji_variation_exclude(_emoji_variation_sequences, exclusions)
Qingqing Deng5e987712019-03-25 16:53:34 -0700538 # Unicode 12.0 adds Basic_Emoji in emoji-sequences.txt. We ignore them here since we are already
539 # checking the emoji presentations with emoji-variation-sequences.txt.
540 # Please refer to http://unicode.org/reports/tr51/#def_basic_emoji_set .
Haibo Huang715857d2020-03-05 11:58:47 -0800541 _emoji_sequences = {k: v for k, v in _emoji_sequences.items() if not v == 'Basic_Emoji' }
Qingqing Deng5e987712019-03-25 16:53:34 -0700542
Siyamed Sinird97df5a2018-04-12 13:11:42 -0700543
544def remove_emoji_variation_exclude(source, items):
545 return source.difference(items.keys())
546
547def remove_emoji_exclude(source, items):
548 return {k: v for k, v in source.items() if k not in items}
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700549
550def flag_sequence(territory_code):
551 return tuple(0x1F1E6 + ord(ch) - ord('A') for ch in territory_code)
552
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700553EQUIVALENT_FLAGS = {
554 flag_sequence('BV'): flag_sequence('NO'),
555 flag_sequence('CP'): flag_sequence('FR'),
556 flag_sequence('HM'): flag_sequence('AU'),
557 flag_sequence('SJ'): flag_sequence('NO'),
558 flag_sequence('UM'): flag_sequence('US'),
559}
560
561COMBINING_KEYCAP = 0x20E3
562
563LEGACY_ANDROID_EMOJI = {
564 0xFE4E5: flag_sequence('JP'),
565 0xFE4E6: flag_sequence('US'),
566 0xFE4E7: flag_sequence('FR'),
567 0xFE4E8: flag_sequence('DE'),
568 0xFE4E9: flag_sequence('IT'),
569 0xFE4EA: flag_sequence('GB'),
570 0xFE4EB: flag_sequence('ES'),
571 0xFE4EC: flag_sequence('RU'),
572 0xFE4ED: flag_sequence('CN'),
573 0xFE4EE: flag_sequence('KR'),
574 0xFE82C: (ord('#'), COMBINING_KEYCAP),
575 0xFE82E: (ord('1'), COMBINING_KEYCAP),
576 0xFE82F: (ord('2'), COMBINING_KEYCAP),
577 0xFE830: (ord('3'), COMBINING_KEYCAP),
578 0xFE831: (ord('4'), COMBINING_KEYCAP),
579 0xFE832: (ord('5'), COMBINING_KEYCAP),
580 0xFE833: (ord('6'), COMBINING_KEYCAP),
581 0xFE834: (ord('7'), COMBINING_KEYCAP),
582 0xFE835: (ord('8'), COMBINING_KEYCAP),
583 0xFE836: (ord('9'), COMBINING_KEYCAP),
584 0xFE837: (ord('0'), COMBINING_KEYCAP),
585}
586
Siyamed Sinir77a1b142018-07-12 12:02:18 -0700587# This is used to define the emoji that should have the same glyph.
588# i.e. previously we had gender based Kiss (0x1F48F), which had the same glyph
589# with Kiss: Woman, Man (0x1F469, 0x200D, 0x2764, 0x200D, 0x1F48B, 0x200D, 0x1F468)
590# in that case a valid row would be:
591# (0x1F469, 0x200D, 0x2764, 0x200D, 0x1F48B, 0x200D, 0x1F468): 0x1F48F,
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700592ZWJ_IDENTICALS = {
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700593}
594
Seigo Nonakac1808632018-05-14 13:39:40 -0700595SAME_FLAG_MAPPINGS = [
596 # Diego Garcia and British Indian Ocean Territory
597 ((0x1F1EE, 0x1F1F4), (0x1F1E9, 0x1F1EC)),
598 # St. Martin and France
599 ((0x1F1F2, 0x1F1EB), (0x1F1EB, 0x1F1F7)),
600 # Spain and Ceuta & Melilla
601 ((0x1F1EA, 0x1F1F8), (0x1F1EA, 0x1F1E6)),
602]
603
Roozbeh Pournaderaa3ee8e2017-04-10 13:52:20 -0700604ZWJ = 0x200D
Doug Feltf874a192016-07-08 17:42:15 -0700605
606def is_fitzpatrick_modifier(cp):
Roozbeh Pournader3b3c78e2016-07-25 14:04:34 -0700607 return 0x1F3FB <= cp <= 0x1F3FF
608
609
610def reverse_emoji(seq):
611 rev = list(reversed(seq))
612 # if there are fitzpatrick modifiers in the sequence, keep them after
613 # the emoji they modify
Haibo Huang715857d2020-03-05 11:58:47 -0800614 for i in range(1, len(rev)):
Roozbeh Pournader3b3c78e2016-07-25 14:04:34 -0700615 if is_fitzpatrick_modifier(rev[i-1]):
616 rev[i], rev[i-1] = rev[i-1], rev[i]
617 return tuple(rev)
Doug Feltf874a192016-07-08 17:42:15 -0700618
619
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700620def compute_expected_emoji():
621 equivalent_emoji = {}
622 sequence_pieces = set()
623 all_sequences = set()
624 all_sequences.update(_emoji_variation_sequences)
625
Raph Levien2b8b8192016-08-09 14:28:54 -0700626 # add zwj sequences not in the current emoji-zwj-sequences.txt
627 adjusted_emoji_zwj_sequences = dict(_emoji_zwj_sequences)
628 adjusted_emoji_zwj_sequences.update(_emoji_zwj_sequences)
Raph Levien2b8b8192016-08-09 14:28:54 -0700629
Roozbeh Pournaderaa3ee8e2017-04-10 13:52:20 -0700630 # Add empty flag tag sequence that is supported as fallback
631 _emoji_sequences[(0x1F3F4, 0xE007F)] = 'Emoji_Tag_Sequence'
632
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700633 for sequence in _emoji_sequences.keys():
634 sequence = tuple(ch for ch in sequence if ch != EMOJI_VS)
635 all_sequences.add(sequence)
636 sequence_pieces.update(sequence)
Roozbeh Pournaderaa3ee8e2017-04-10 13:52:20 -0700637 if _emoji_sequences.get(sequence, None) == 'Emoji_Tag_Sequence':
Roozbeh Pournader63d4d0d2017-05-18 18:38:36 -0700638 # Add reverse of all emoji ZWJ sequences, which are added to the
639 # fonts as a workaround to get the sequences work in RTL text.
Roozbeh Pournaderaa3ee8e2017-04-10 13:52:20 -0700640 # TODO: test if these are actually needed by Minikin/HarfBuzz.
641 reversed_seq = reverse_emoji(sequence)
642 all_sequences.add(reversed_seq)
643 equivalent_emoji[reversed_seq] = sequence
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700644
Raph Levien2b8b8192016-08-09 14:28:54 -0700645 for sequence in adjusted_emoji_zwj_sequences.keys():
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700646 sequence = tuple(ch for ch in sequence if ch != EMOJI_VS)
647 all_sequences.add(sequence)
648 sequence_pieces.update(sequence)
649 # Add reverse of all emoji ZWJ sequences, which are added to the fonts
650 # as a workaround to get the sequences work in RTL text.
Roozbeh Pournader3b3c78e2016-07-25 14:04:34 -0700651 reversed_seq = reverse_emoji(sequence)
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700652 all_sequences.add(reversed_seq)
653 equivalent_emoji[reversed_seq] = sequence
654
Seigo Nonakac1808632018-05-14 13:39:40 -0700655 for first, second in SAME_FLAG_MAPPINGS:
656 equivalent_emoji[first] = second
657
Roozbeh Pournaderaa3ee8e2017-04-10 13:52:20 -0700658 # Add all tag characters used in flags
659 sequence_pieces.update(range(0xE0030, 0xE0039 + 1))
660 sequence_pieces.update(range(0xE0061, 0xE007A + 1))
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700661
662 all_emoji = (
663 _emoji_properties['Emoji'] |
664 all_sequences |
665 sequence_pieces |
666 set(LEGACY_ANDROID_EMOJI.keys()))
667 default_emoji = (
668 _emoji_properties['Emoji_Presentation'] |
669 all_sequences |
670 set(LEGACY_ANDROID_EMOJI.keys()))
671
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700672 equivalent_emoji.update(EQUIVALENT_FLAGS)
673 equivalent_emoji.update(LEGACY_ANDROID_EMOJI)
674 equivalent_emoji.update(ZWJ_IDENTICALS)
Roozbeh Pournaderaa3ee8e2017-04-10 13:52:20 -0700675
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700676 for seq in _emoji_variation_sequences:
677 equivalent_emoji[seq] = seq[0]
678
679 return all_emoji, default_emoji, equivalent_emoji
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700680
681
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700682def check_compact_only_fallback():
Haibo Huang715857d2020-03-05 11:58:47 -0800683 for name, fallback_chain in _fallback_chains.items():
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700684 for record in fallback_chain:
685 if record.variant == 'compact':
686 same_script_elegants = [x for x in fallback_chain
687 if x.scripts == record.scripts and x.variant == 'elegant']
688 assert same_script_elegants, (
689 '%s must be in elegant of %s as fallback of "%s" too' % (
690 record.font, record.scripts, record.fallback_for),)
691
692
Roozbeh Pournaderbac1aec2016-07-27 13:08:37 -0700693def check_vertical_metrics():
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700694 for record in _all_fonts:
Roozbeh Pournaderbac1aec2016-07-27 13:08:37 -0700695 if record.name in ['sans-serif', 'sans-serif-condensed']:
696 font = open_font(record.font)
Roozbeh Pournaderede3a172016-07-27 16:35:12 -0700697 assert font['head'].yMax == 2163 and font['head'].yMin == -555, (
Roozbeh Pournader63d4d0d2017-05-18 18:38:36 -0700698 'yMax and yMin of %s do not match expected values.' % (
699 record.font,))
Roozbeh Pournaderede3a172016-07-27 16:35:12 -0700700
Roozbeh Pournader63d4d0d2017-05-18 18:38:36 -0700701 if record.name in ['sans-serif', 'sans-serif-condensed',
702 'serif', 'monospace']:
Roozbeh Pournaderede3a172016-07-27 16:35:12 -0700703 font = open_font(record.font)
Roozbeh Pournader63d4d0d2017-05-18 18:38:36 -0700704 assert (font['hhea'].ascent == 1900 and
705 font['hhea'].descent == -500), (
706 'ascent and descent of %s do not match expected '
707 'values.' % (record.font,))
708
709
710def check_cjk_punctuation():
711 cjk_scripts = {'Hans', 'Hant', 'Jpan', 'Kore'}
712 cjk_punctuation = range(0x3000, 0x301F + 1)
Haibo Huang715857d2020-03-05 11:58:47 -0800713 for name, fallback_chain in _fallback_chains.items():
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700714 for record in fallback_chain:
715 if record.scripts.intersection(cjk_scripts):
716 # CJK font seen. Stop checking the rest of the fonts.
717 break
718 assert_font_supports_none_of_chars(record.font, cjk_punctuation, name)
Roozbeh Pournaderbac1aec2016-07-27 13:08:37 -0700719
Seigo Nonaka56811242021-04-16 00:11:43 -0700720def getPostScriptName(font):
721 font_file, index = font
722 font_path = path.join(_fonts_dir, font_file)
723 if index is not None:
724 # Use the first font file in the collection for resolving post script name.
725 ttf = ttLib.TTFont(font_path, fontNumber=0)
726 else:
727 ttf = ttLib.TTFont(font_path)
728
729 nameTable = ttf['name']
730 for name in nameTable.names:
731 if (name.nameID == 6 and name.platformID == 3 and name.platEncID == 1
732 and name.langID == 0x0409):
733 return str(name)
734
735def check_canonical_name():
736 for record in _all_fonts:
737 file_name, index = record.font
738
739 psName = getPostScriptName(record.font)
740 if record.psName:
741 # If fonts element has postScriptName attribute, it should match with the PostScript
742 # name in the name table.
743 assert psName == record.psName, ('postScriptName attribute %s should match with %s' % (
744 record.psName, psName))
745 else:
746 # If fonts element doesn't have postScriptName attribute, the file name should match
747 # with the PostScript name in the name table.
748 assert psName == file_name[:-4], ('file name %s should match with %s' % (
749 file_name, psName))
750
Roozbeh Pournaderbac1aec2016-07-27 13:08:37 -0700751
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800752def main():
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800753 global _fonts_dir
Doug Feltf874a192016-07-08 17:42:15 -0700754 target_out = sys.argv[1]
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800755 _fonts_dir = path.join(target_out, 'fonts')
756
757 fonts_xml_path = path.join(target_out, 'etc', 'fonts.xml')
758 parse_fonts_xml(fonts_xml_path)
759
Seigo Nonaka99a7b602017-07-05 16:06:23 -0700760 check_compact_only_fallback()
761
Roozbeh Pournaderbac1aec2016-07-27 13:08:37 -0700762 check_vertical_metrics()
763
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800764 hyphens_dir = path.join(target_out, 'usr', 'hyphen-data')
765 check_hyphens(hyphens_dir)
766
Roozbeh Pournader63d4d0d2017-05-18 18:38:36 -0700767 check_cjk_punctuation()
768
Seigo Nonaka56811242021-04-16 00:11:43 -0700769 check_canonical_name()
770
Roozbeh Pournader27ec3ac2016-03-31 13:05:32 -0700771 check_emoji = sys.argv[2]
772 if check_emoji == 'true':
773 ucd_path = sys.argv[3]
774 parse_ucd(ucd_path)
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700775 all_emoji, default_emoji, equivalent_emoji = compute_expected_emoji()
Seigo Nonaka2e9e7922021-05-04 17:12:35 -0700776 check_emoji_compat()
Roozbeh Pournader5dde0872016-03-31 13:54:56 -0700777 check_emoji_coverage(all_emoji, equivalent_emoji)
778 check_emoji_defaults(default_emoji)
Roozbeh Pournaderfa1facc2016-03-16 13:53:47 -0700779
Roozbeh Pournader0e969e22016-03-09 23:08:45 -0800780
781if __name__ == '__main__':
782 main()