blob: 0821dc62cb56aff98313a282affb7f97dc007353 [file] [log] [blame]
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001/* vi:set ts=8 sts=4 sw=4 noet:
2 *
3 * VIM - Vi IMproved by Bram Moolenaar
4 *
5 * Do ":help uganda" in Vim to read copying and usage conditions.
6 * Do ":help credits" in Vim to see a list of people who contributed.
7 * See README.txt for an overview of the Vim source code.
8 */
9
10/*
11 * spellsuggest.c: functions for spelling suggestions
12 */
13
14#include "vim.h"
15
16#if defined(FEAT_SPELL) || defined(PROTO)
17
18/*
19 * Use this to adjust the score after finding suggestions, based on the
20 * suggested word sounding like the bad word. This is much faster than doing
21 * it for every possible suggestion.
22 * Disadvantage: When "the" is typed as "hte" it sounds quite different ("@"
23 * vs "ht") and goes down in the list.
24 * Used when 'spellsuggest' is set to "best".
25 */
26#define RESCORE(word_score, sound_score) ((3 * word_score + sound_score) / 4)
27
28/*
29 * Do the opposite: based on a maximum end score and a known sound score,
30 * compute the maximum word score that can be used.
31 */
32#define MAXSCORE(word_score, sound_score) ((4 * word_score - sound_score) / 3)
33
34// only used for su_badflags
35#define WF_MIXCAP 0x20 // mix of upper and lower case: macaRONI
36
37/*
38 * Information used when looking for suggestions.
39 */
40typedef struct suginfo_S
41{
42 garray_T su_ga; // suggestions, contains "suggest_T"
43 int su_maxcount; // max. number of suggestions displayed
44 int su_maxscore; // maximum score for adding to su_ga
45 int su_sfmaxscore; // idem, for when doing soundfold words
46 garray_T su_sga; // like su_ga, sound-folded scoring
47 char_u *su_badptr; // start of bad word in line
48 int su_badlen; // length of detected bad word in line
49 int su_badflags; // caps flags for bad word
50 char_u su_badword[MAXWLEN]; // bad word truncated at su_badlen
51 char_u su_fbadword[MAXWLEN]; // su_badword case-folded
52 char_u su_sal_badword[MAXWLEN]; // su_badword soundfolded
53 hashtab_T su_banned; // table with banned words
54 slang_T *su_sallang; // default language for sound folding
55} suginfo_T;
56
57// One word suggestion. Used in "si_ga".
58typedef struct suggest_S
59{
60 char_u *st_word; // suggested word, allocated string
61 int st_wordlen; // STRLEN(st_word)
62 int st_orglen; // length of replaced text
63 int st_score; // lower is better
64 int st_altscore; // used when st_score compares equal
65 int st_salscore; // st_score is for soundalike
66 int st_had_bonus; // bonus already included in score
67 slang_T *st_slang; // language used for sound folding
68} suggest_T;
69
70#define SUG(ga, i) (((suggest_T *)(ga).ga_data)[i])
71
72// TRUE if a word appears in the list of banned words.
73#define WAS_BANNED(su, word) (!HASHITEM_EMPTY(hash_find(&su->su_banned, word)))
74
75// Number of suggestions kept when cleaning up. We need to keep more than
76// what is displayed, because when rescore_suggestions() is called the score
77// may change and wrong suggestions may be removed later.
78#define SUG_CLEAN_COUNT(su) ((su)->su_maxcount < 130 ? 150 : (su)->su_maxcount + 20)
79
80// Threshold for sorting and cleaning up suggestions. Don't want to keep lots
81// of suggestions that are not going to be displayed.
82#define SUG_MAX_COUNT(su) (SUG_CLEAN_COUNT(su) + 50)
83
84// score for various changes
85#define SCORE_SPLIT 149 // split bad word
86#define SCORE_SPLIT_NO 249 // split bad word with NOSPLITSUGS
87#define SCORE_ICASE 52 // slightly different case
88#define SCORE_REGION 200 // word is for different region
89#define SCORE_RARE 180 // rare word
90#define SCORE_SWAP 75 // swap two characters
91#define SCORE_SWAP3 110 // swap two characters in three
92#define SCORE_REP 65 // REP replacement
93#define SCORE_SUBST 93 // substitute a character
94#define SCORE_SIMILAR 33 // substitute a similar character
95#define SCORE_SUBCOMP 33 // substitute a composing character
96#define SCORE_DEL 94 // delete a character
97#define SCORE_DELDUP 66 // delete a duplicated character
98#define SCORE_DELCOMP 28 // delete a composing character
99#define SCORE_INS 96 // insert a character
100#define SCORE_INSDUP 67 // insert a duplicate character
101#define SCORE_INSCOMP 30 // insert a composing character
102#define SCORE_NONWORD 103 // change non-word to word char
103
104#define SCORE_FILE 30 // suggestion from a file
105#define SCORE_MAXINIT 350 // Initial maximum score: higher == slower.
106 // 350 allows for about three changes.
107
108#define SCORE_COMMON1 30 // subtracted for words seen before
109#define SCORE_COMMON2 40 // subtracted for words often seen
110#define SCORE_COMMON3 50 // subtracted for words very often seen
111#define SCORE_THRES2 10 // word count threshold for COMMON2
112#define SCORE_THRES3 100 // word count threshold for COMMON3
113
114// When trying changed soundfold words it becomes slow when trying more than
115// two changes. With less then two changes it's slightly faster but we miss a
116// few good suggestions. In rare cases we need to try three of four changes.
117#define SCORE_SFMAX1 200 // maximum score for first try
118#define SCORE_SFMAX2 300 // maximum score for second try
119#define SCORE_SFMAX3 400 // maximum score for third try
120
121#define SCORE_BIG SCORE_INS * 3 // big difference
122#define SCORE_MAXMAX 999999 // accept any score
123#define SCORE_LIMITMAX 350 // for spell_edit_score_limit()
124
125// for spell_edit_score_limit() we need to know the minimum value of
126// SCORE_ICASE, SCORE_SWAP, SCORE_DEL, SCORE_SIMILAR and SCORE_INS
127#define SCORE_EDIT_MIN SCORE_SIMILAR
128
129/*
130 * For finding suggestions: At each node in the tree these states are tried:
131 */
132typedef enum
133{
134 STATE_START = 0, // At start of node check for NUL bytes (goodword
135 // ends); if badword ends there is a match, otherwise
136 // try splitting word.
137 STATE_NOPREFIX, // try without prefix
138 STATE_SPLITUNDO, // Undo splitting.
139 STATE_ENDNUL, // Past NUL bytes at start of the node.
140 STATE_PLAIN, // Use each byte of the node.
141 STATE_DEL, // Delete a byte from the bad word.
142 STATE_INS_PREP, // Prepare for inserting bytes.
143 STATE_INS, // Insert a byte in the bad word.
144 STATE_SWAP, // Swap two bytes.
145 STATE_UNSWAP, // Undo swap two characters.
146 STATE_SWAP3, // Swap two characters over three.
147 STATE_UNSWAP3, // Undo Swap two characters over three.
148 STATE_UNROT3L, // Undo rotate three characters left
149 STATE_UNROT3R, // Undo rotate three characters right
150 STATE_REP_INI, // Prepare for using REP items.
151 STATE_REP, // Use matching REP items from the .aff file.
152 STATE_REP_UNDO, // Undo a REP item replacement.
153 STATE_FINAL // End of this node.
154} state_T;
155
156/*
157 * Struct to keep the state at each level in suggest_try_change().
158 */
159typedef struct trystate_S
160{
161 state_T ts_state; // state at this level, STATE_
162 int ts_score; // score
163 idx_T ts_arridx; // index in tree array, start of node
164 short ts_curi; // index in list of child nodes
165 char_u ts_fidx; // index in fword[], case-folded bad word
166 char_u ts_fidxtry; // ts_fidx at which bytes may be changed
167 char_u ts_twordlen; // valid length of tword[]
168 char_u ts_prefixdepth; // stack depth for end of prefix or
169 // PFD_PREFIXTREE or PFD_NOPREFIX
170 char_u ts_flags; // TSF_ flags
171 char_u ts_tcharlen; // number of bytes in tword character
172 char_u ts_tcharidx; // current byte index in tword character
173 char_u ts_isdiff; // DIFF_ values
174 char_u ts_fcharstart; // index in fword where badword char started
175 char_u ts_prewordlen; // length of word in "preword[]"
176 char_u ts_splitoff; // index in "tword" after last split
177 char_u ts_splitfidx; // "ts_fidx" at word split
178 char_u ts_complen; // nr of compound words used
179 char_u ts_compsplit; // index for "compflags" where word was spit
180 char_u ts_save_badflags; // su_badflags saved here
181 char_u ts_delidx; // index in fword for char that was deleted,
182 // valid when "ts_flags" has TSF_DIDDEL
183} trystate_T;
184
185// values for ts_isdiff
186#define DIFF_NONE 0 // no different byte (yet)
187#define DIFF_YES 1 // different byte found
188#define DIFF_INSERT 2 // inserting character
189
190// values for ts_flags
191#define TSF_PREFIXOK 1 // already checked that prefix is OK
192#define TSF_DIDSPLIT 2 // tried split at this point
193#define TSF_DIDDEL 4 // did a delete, "ts_delidx" has index
194
195// special values ts_prefixdepth
196#define PFD_NOPREFIX 0xff // not using prefixes
197#define PFD_PREFIXTREE 0xfe // walking through the prefix tree
198#define PFD_NOTSPECIAL 0xfd // highest value that's not special
199
200static void spell_find_suggest(char_u *badptr, int badlen, suginfo_T *su, int maxcount, int banbadword, int need_cap, int interactive);
201#ifdef FEAT_EVAL
202static void spell_suggest_expr(suginfo_T *su, char_u *expr);
203#endif
204static void spell_suggest_file(suginfo_T *su, char_u *fname);
205static void spell_suggest_intern(suginfo_T *su, int interactive);
206static void spell_find_cleanup(suginfo_T *su);
207static void suggest_try_special(suginfo_T *su);
208static void suggest_try_change(suginfo_T *su);
209static void suggest_trie_walk(suginfo_T *su, langp_T *lp, char_u *fword, int soundfold);
210static void go_deeper(trystate_T *stack, int depth, int score_add);
211static void find_keepcap_word(slang_T *slang, char_u *fword, char_u *kword);
212static void score_comp_sal(suginfo_T *su);
213static void score_combine(suginfo_T *su);
214static int stp_sal_score(suggest_T *stp, suginfo_T *su, slang_T *slang, char_u *badsound);
215static void suggest_try_soundalike_prep(void);
216static void suggest_try_soundalike(suginfo_T *su);
217static void suggest_try_soundalike_finish(void);
218static void add_sound_suggest(suginfo_T *su, char_u *goodword, int score, langp_T *lp);
219static int soundfold_find(slang_T *slang, char_u *word);
220static int similar_chars(slang_T *slang, int c1, int c2);
221static void add_suggestion(suginfo_T *su, garray_T *gap, char_u *goodword, int badlen, int score, int altscore, int had_bonus, slang_T *slang, int maxsf);
222static void check_suggestions(suginfo_T *su, garray_T *gap);
223static void add_banned(suginfo_T *su, char_u *word);
224static void rescore_suggestions(suginfo_T *su);
225static void rescore_one(suginfo_T *su, suggest_T *stp);
226static int cleanup_suggestions(garray_T *gap, int maxscore, int keep);
227static int soundalike_score(char_u *goodsound, char_u *badsound);
228static int spell_edit_score(slang_T *slang, char_u *badword, char_u *goodword);
229static int spell_edit_score_limit(slang_T *slang, char_u *badword, char_u *goodword, int limit);
230static int spell_edit_score_limit_w(slang_T *slang, char_u *badword, char_u *goodword, int limit);
231
232/*
233 * Return TRUE when the sequence of flags in "compflags" plus "flag" can
234 * possibly form a valid compounded word. This also checks the COMPOUNDRULE
235 * lines if they don't contain wildcards.
236 */
237 static int
238can_be_compound(
239 trystate_T *sp,
240 slang_T *slang,
241 char_u *compflags,
242 int flag)
243{
244 // If the flag doesn't appear in sl_compstartflags or sl_compallflags
245 // then it can't possibly compound.
246 if (!byte_in_str(sp->ts_complen == sp->ts_compsplit
247 ? slang->sl_compstartflags : slang->sl_compallflags, flag))
248 return FALSE;
249
250 // If there are no wildcards, we can check if the flags collected so far
251 // possibly can form a match with COMPOUNDRULE patterns. This only
252 // makes sense when we have two or more words.
253 if (slang->sl_comprules != NULL && sp->ts_complen > sp->ts_compsplit)
254 {
255 int v;
256
257 compflags[sp->ts_complen] = flag;
258 compflags[sp->ts_complen + 1] = NUL;
259 v = match_compoundrule(slang, compflags + sp->ts_compsplit);
260 compflags[sp->ts_complen] = NUL;
261 return v;
262 }
263
264 return TRUE;
265}
266
267/*
268 * Adjust the score of common words.
269 */
270 static int
271score_wordcount_adj(
272 slang_T *slang,
273 int score,
274 char_u *word,
275 int split) // word was split, less bonus
276{
277 hashitem_T *hi;
278 wordcount_T *wc;
279 int bonus;
280 int newscore;
281
282 hi = hash_find(&slang->sl_wordcount, word);
283 if (!HASHITEM_EMPTY(hi))
284 {
285 wc = HI2WC(hi);
286 if (wc->wc_count < SCORE_THRES2)
287 bonus = SCORE_COMMON1;
288 else if (wc->wc_count < SCORE_THRES3)
289 bonus = SCORE_COMMON2;
290 else
291 bonus = SCORE_COMMON3;
292 if (split)
293 newscore = score - bonus / 2;
294 else
295 newscore = score - bonus;
296 if (newscore < 0)
297 return 0;
298 return newscore;
299 }
300 return score;
301}
302
303/*
304 * Like captype() but for a KEEPCAP word add ONECAP if the word starts with a
305 * capital. So that make_case_word() can turn WOrd into Word.
306 * Add ALLCAP for "WOrD".
307 */
308 static int
309badword_captype(char_u *word, char_u *end)
310{
311 int flags = captype(word, end);
312 int c;
313 int l, u;
314 int first;
315 char_u *p;
316
317 if (flags & WF_KEEPCAP)
318 {
319 // Count the number of UPPER and lower case letters.
320 l = u = 0;
321 first = FALSE;
322 for (p = word; p < end; MB_PTR_ADV(p))
323 {
324 c = PTR2CHAR(p);
325 if (SPELL_ISUPPER(c))
326 {
327 ++u;
328 if (p == word)
329 first = TRUE;
330 }
331 else
332 ++l;
333 }
334
335 // If there are more UPPER than lower case letters suggest an
336 // ALLCAP word. Otherwise, if the first letter is UPPER then
337 // suggest ONECAP. Exception: "ALl" most likely should be "All",
338 // require three upper case letters.
339 if (u > l && u > 2)
340 flags |= WF_ALLCAP;
341 else if (first)
342 flags |= WF_ONECAP;
343
344 if (u >= 2 && l >= 2) // maCARONI maCAroni
345 flags |= WF_MIXCAP;
346 }
347 return flags;
348}
349
350/*
351 * Opposite of offset2bytes().
352 * "pp" points to the bytes and is advanced over it.
353 * Returns the offset.
354 */
355 static int
356bytes2offset(char_u **pp)
357{
358 char_u *p = *pp;
359 int nr;
360 int c;
361
362 c = *p++;
363 if ((c & 0x80) == 0x00) // 1 byte
364 {
365 nr = c - 1;
366 }
367 else if ((c & 0xc0) == 0x80) // 2 bytes
368 {
369 nr = (c & 0x3f) - 1;
370 nr = nr * 255 + (*p++ - 1);
371 }
372 else if ((c & 0xe0) == 0xc0) // 3 bytes
373 {
374 nr = (c & 0x1f) - 1;
375 nr = nr * 255 + (*p++ - 1);
376 nr = nr * 255 + (*p++ - 1);
377 }
378 else // 4 bytes
379 {
380 nr = (c & 0x0f) - 1;
381 nr = nr * 255 + (*p++ - 1);
382 nr = nr * 255 + (*p++ - 1);
383 nr = nr * 255 + (*p++ - 1);
384 }
385
386 *pp = p;
387 return nr;
388}
389
390// values for sps_flags
391#define SPS_BEST 1
392#define SPS_FAST 2
393#define SPS_DOUBLE 4
394
395static int sps_flags = SPS_BEST; // flags from 'spellsuggest'
396static int sps_limit = 9999; // max nr of suggestions given
397
398/*
399 * Check the 'spellsuggest' option. Return FAIL if it's wrong.
400 * Sets "sps_flags" and "sps_limit".
401 */
402 int
403spell_check_sps(void)
404{
405 char_u *p;
406 char_u *s;
407 char_u buf[MAXPATHL];
408 int f;
409
410 sps_flags = 0;
411 sps_limit = 9999;
412
413 for (p = p_sps; *p != NUL; )
414 {
415 copy_option_part(&p, buf, MAXPATHL, ",");
416
417 f = 0;
418 if (VIM_ISDIGIT(*buf))
419 {
420 s = buf;
421 sps_limit = getdigits(&s);
422 if (*s != NUL && !VIM_ISDIGIT(*s))
423 f = -1;
424 }
425 else if (STRCMP(buf, "best") == 0)
426 f = SPS_BEST;
427 else if (STRCMP(buf, "fast") == 0)
428 f = SPS_FAST;
429 else if (STRCMP(buf, "double") == 0)
430 f = SPS_DOUBLE;
431 else if (STRNCMP(buf, "expr:", 5) != 0
432 && STRNCMP(buf, "file:", 5) != 0)
433 f = -1;
434
435 if (f == -1 || (sps_flags != 0 && f != 0))
436 {
437 sps_flags = SPS_BEST;
438 sps_limit = 9999;
439 return FAIL;
440 }
441 if (f != 0)
442 sps_flags = f;
443 }
444
445 if (sps_flags == 0)
446 sps_flags = SPS_BEST;
447
448 return OK;
449}
450
451/*
452 * "z=": Find badly spelled word under or after the cursor.
453 * Give suggestions for the properly spelled word.
454 * In Visual mode use the highlighted word as the bad word.
455 * When "count" is non-zero use that suggestion.
456 */
457 void
458spell_suggest(int count)
459{
460 char_u *line;
461 pos_T prev_cursor = curwin->w_cursor;
462 char_u wcopy[MAXWLEN + 2];
463 char_u *p;
464 int i;
465 int c;
466 suginfo_T sug;
467 suggest_T *stp;
468 int mouse_used;
469 int need_cap;
470 int limit;
471 int selected = count;
472 int badlen = 0;
473 int msg_scroll_save = msg_scroll;
Bram Moolenaar152e79e2020-06-10 15:32:08 +0200474 int wo_spell_save = curwin->w_p_spell;
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200475
Bram Moolenaar152e79e2020-06-10 15:32:08 +0200476 if (!curwin->w_p_spell)
477 {
478 did_set_spelllang(curwin);
479 curwin->w_p_spell = TRUE;
480 }
481
482 if (*curwin->w_s->b_p_spl == NUL)
483 {
484 emsg(_(e_no_spell));
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200485 return;
Bram Moolenaar152e79e2020-06-10 15:32:08 +0200486 }
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200487
488 if (VIsual_active)
489 {
490 // Use the Visually selected text as the bad word. But reject
491 // a multi-line selection.
492 if (curwin->w_cursor.lnum != VIsual.lnum)
493 {
494 vim_beep(BO_SPELL);
495 return;
496 }
497 badlen = (int)curwin->w_cursor.col - (int)VIsual.col;
498 if (badlen < 0)
499 badlen = -badlen;
500 else
501 curwin->w_cursor.col = VIsual.col;
502 ++badlen;
503 end_visual_mode();
504 }
505 // Find the start of the badly spelled word.
506 else if (spell_move_to(curwin, FORWARD, TRUE, TRUE, NULL) == 0
507 || curwin->w_cursor.col > prev_cursor.col)
508 {
509 // No bad word or it starts after the cursor: use the word under the
510 // cursor.
511 curwin->w_cursor = prev_cursor;
512 line = ml_get_curline();
513 p = line + curwin->w_cursor.col;
514 // Backup to before start of word.
515 while (p > line && spell_iswordp_nmw(p, curwin))
516 MB_PTR_BACK(line, p);
517 // Forward to start of word.
518 while (*p != NUL && !spell_iswordp_nmw(p, curwin))
519 MB_PTR_ADV(p);
520
521 if (!spell_iswordp_nmw(p, curwin)) // No word found.
522 {
523 beep_flush();
524 return;
525 }
526 curwin->w_cursor.col = (colnr_T)(p - line);
527 }
528
529 // Get the word and its length.
530
531 // Figure out if the word should be capitalised.
532 need_cap = check_need_cap(curwin->w_cursor.lnum, curwin->w_cursor.col);
533
534 // Make a copy of current line since autocommands may free the line.
535 line = vim_strsave(ml_get_curline());
536 if (line == NULL)
537 goto skip;
538
539 // Get the list of suggestions. Limit to 'lines' - 2 or the number in
540 // 'spellsuggest', whatever is smaller.
541 if (sps_limit > (int)Rows - 2)
542 limit = (int)Rows - 2;
543 else
544 limit = sps_limit;
545 spell_find_suggest(line + curwin->w_cursor.col, badlen, &sug, limit,
546 TRUE, need_cap, TRUE);
547
548 if (sug.su_ga.ga_len == 0)
549 msg(_("Sorry, no suggestions"));
550 else if (count > 0)
551 {
552 if (count > sug.su_ga.ga_len)
Bram Moolenaar6c52f822019-12-25 14:13:03 +0100553 smsg(_("Sorry, only %ld suggestions"), (long)sug.su_ga.ga_len);
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200554 }
555 else
556 {
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200557#ifdef FEAT_RIGHTLEFT
558 // When 'rightleft' is set the list is drawn right-left.
559 cmdmsg_rl = curwin->w_p_rl;
560 if (cmdmsg_rl)
561 msg_col = Columns - 1;
562#endif
563
564 // List the suggestions.
565 msg_start();
566 msg_row = Rows - 1; // for when 'cmdheight' > 1
567 lines_left = Rows; // avoid more prompt
568 vim_snprintf((char *)IObuff, IOSIZE, _("Change \"%.*s\" to:"),
569 sug.su_badlen, sug.su_badptr);
570#ifdef FEAT_RIGHTLEFT
571 if (cmdmsg_rl && STRNCMP(IObuff, "Change", 6) == 0)
572 {
573 // And now the rabbit from the high hat: Avoid showing the
574 // untranslated message rightleft.
575 vim_snprintf((char *)IObuff, IOSIZE, ":ot \"%.*s\" egnahC",
576 sug.su_badlen, sug.su_badptr);
577 }
578#endif
579 msg_puts((char *)IObuff);
580 msg_clr_eos();
581 msg_putchar('\n');
582
583 msg_scroll = TRUE;
584 for (i = 0; i < sug.su_ga.ga_len; ++i)
585 {
586 stp = &SUG(sug.su_ga, i);
587
588 // The suggested word may replace only part of the bad word, add
589 // the not replaced part.
590 vim_strncpy(wcopy, stp->st_word, MAXWLEN);
591 if (sug.su_badlen > stp->st_orglen)
592 vim_strncpy(wcopy + stp->st_wordlen,
593 sug.su_badptr + stp->st_orglen,
594 sug.su_badlen - stp->st_orglen);
595 vim_snprintf((char *)IObuff, IOSIZE, "%2d", i + 1);
596#ifdef FEAT_RIGHTLEFT
597 if (cmdmsg_rl)
598 rl_mirror(IObuff);
599#endif
600 msg_puts((char *)IObuff);
601
602 vim_snprintf((char *)IObuff, IOSIZE, " \"%s\"", wcopy);
603 msg_puts((char *)IObuff);
604
605 // The word may replace more than "su_badlen".
606 if (sug.su_badlen < stp->st_orglen)
607 {
608 vim_snprintf((char *)IObuff, IOSIZE, _(" < \"%.*s\""),
609 stp->st_orglen, sug.su_badptr);
610 msg_puts((char *)IObuff);
611 }
612
613 if (p_verbose > 0)
614 {
615 // Add the score.
616 if (sps_flags & (SPS_DOUBLE | SPS_BEST))
617 vim_snprintf((char *)IObuff, IOSIZE, " (%s%d - %d)",
618 stp->st_salscore ? "s " : "",
619 stp->st_score, stp->st_altscore);
620 else
621 vim_snprintf((char *)IObuff, IOSIZE, " (%d)",
622 stp->st_score);
623#ifdef FEAT_RIGHTLEFT
624 if (cmdmsg_rl)
625 // Mirror the numbers, but keep the leading space.
626 rl_mirror(IObuff + 1);
627#endif
628 msg_advance(30);
629 msg_puts((char *)IObuff);
630 }
631 msg_putchar('\n');
632 }
633
634#ifdef FEAT_RIGHTLEFT
635 cmdmsg_rl = FALSE;
636 msg_col = 0;
637#endif
638 // Ask for choice.
639 selected = prompt_for_number(&mouse_used);
640 if (mouse_used)
641 selected -= lines_left;
642 lines_left = Rows; // avoid more prompt
643 // don't delay for 'smd' in normal_cmd()
644 msg_scroll = msg_scroll_save;
645 }
646
647 if (selected > 0 && selected <= sug.su_ga.ga_len && u_save_cursor() == OK)
648 {
649 // Save the from and to text for :spellrepall.
Bram Moolenaar6c52f822019-12-25 14:13:03 +0100650 VIM_CLEAR(repl_from);
651 VIM_CLEAR(repl_to);
652
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200653 stp = &SUG(sug.su_ga, selected - 1);
654 if (sug.su_badlen > stp->st_orglen)
655 {
656 // Replacing less than "su_badlen", append the remainder to
657 // repl_to.
658 repl_from = vim_strnsave(sug.su_badptr, sug.su_badlen);
659 vim_snprintf((char *)IObuff, IOSIZE, "%s%.*s", stp->st_word,
660 sug.su_badlen - stp->st_orglen,
661 sug.su_badptr + stp->st_orglen);
662 repl_to = vim_strsave(IObuff);
663 }
664 else
665 {
666 // Replacing su_badlen or more, use the whole word.
667 repl_from = vim_strnsave(sug.su_badptr, stp->st_orglen);
668 repl_to = vim_strsave(stp->st_word);
669 }
670
671 // Replace the word.
672 p = alloc(STRLEN(line) - stp->st_orglen + stp->st_wordlen + 1);
673 if (p != NULL)
674 {
675 c = (int)(sug.su_badptr - line);
676 mch_memmove(p, line, c);
677 STRCPY(p + c, stp->st_word);
678 STRCAT(p, sug.su_badptr + stp->st_orglen);
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200679
680 // For redo we use a change-word command.
681 ResetRedobuff();
682 AppendToRedobuff((char_u *)"ciw");
683 AppendToRedobuffLit(p + c,
684 stp->st_wordlen + sug.su_badlen - stp->st_orglen);
685 AppendCharToRedobuff(ESC);
686
Bram Moolenaar6b949612020-06-29 23:18:42 +0200687 // "p" may be freed here
688 ml_replace(curwin->w_cursor.lnum, p, FALSE);
689 curwin->w_cursor.col = c;
690
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200691 changed_bytes(curwin->w_cursor.lnum, c);
692 }
693 }
694 else
695 curwin->w_cursor = prev_cursor;
696
697 spell_find_cleanup(&sug);
698skip:
699 vim_free(line);
Bram Moolenaar152e79e2020-06-10 15:32:08 +0200700 curwin->w_p_spell = wo_spell_save;
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200701}
702
703/*
704 * Find spell suggestions for "word". Return them in the growarray "*gap" as
705 * a list of allocated strings.
706 */
707 void
708spell_suggest_list(
709 garray_T *gap,
710 char_u *word,
711 int maxcount, // maximum nr of suggestions
712 int need_cap, // 'spellcapcheck' matched
713 int interactive)
714{
715 suginfo_T sug;
716 int i;
717 suggest_T *stp;
718 char_u *wcopy;
719
720 spell_find_suggest(word, 0, &sug, maxcount, FALSE, need_cap, interactive);
721
722 // Make room in "gap".
723 ga_init2(gap, sizeof(char_u *), sug.su_ga.ga_len + 1);
724 if (ga_grow(gap, sug.su_ga.ga_len) == OK)
725 {
726 for (i = 0; i < sug.su_ga.ga_len; ++i)
727 {
728 stp = &SUG(sug.su_ga, i);
729
730 // The suggested word may replace only part of "word", add the not
731 // replaced part.
732 wcopy = alloc(stp->st_wordlen
733 + (unsigned)STRLEN(sug.su_badptr + stp->st_orglen) + 1);
734 if (wcopy == NULL)
735 break;
736 STRCPY(wcopy, stp->st_word);
737 STRCPY(wcopy + stp->st_wordlen, sug.su_badptr + stp->st_orglen);
738 ((char_u **)gap->ga_data)[gap->ga_len++] = wcopy;
739 }
740 }
741
742 spell_find_cleanup(&sug);
743}
744
745/*
746 * Find spell suggestions for the word at the start of "badptr".
747 * Return the suggestions in "su->su_ga".
748 * The maximum number of suggestions is "maxcount".
749 * Note: does use info for the current window.
750 * This is based on the mechanisms of Aspell, but completely reimplemented.
751 */
752 static void
753spell_find_suggest(
754 char_u *badptr,
755 int badlen, // length of bad word or 0 if unknown
756 suginfo_T *su,
757 int maxcount,
758 int banbadword, // don't include badword in suggestions
759 int need_cap, // word should start with capital
760 int interactive)
761{
762 hlf_T attr = HLF_COUNT;
763 char_u buf[MAXPATHL];
764 char_u *p;
765 int do_combine = FALSE;
766 char_u *sps_copy;
767#ifdef FEAT_EVAL
768 static int expr_busy = FALSE;
769#endif
770 int c;
771 int i;
772 langp_T *lp;
773
774 // Set the info in "*su".
Bram Moolenaara80faa82020-04-12 19:37:17 +0200775 CLEAR_POINTER(su);
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200776 ga_init2(&su->su_ga, (int)sizeof(suggest_T), 10);
777 ga_init2(&su->su_sga, (int)sizeof(suggest_T), 10);
778 if (*badptr == NUL)
779 return;
780 hash_init(&su->su_banned);
781
782 su->su_badptr = badptr;
783 if (badlen != 0)
784 su->su_badlen = badlen;
785 else
786 su->su_badlen = spell_check(curwin, su->su_badptr, &attr, NULL, FALSE);
787 su->su_maxcount = maxcount;
788 su->su_maxscore = SCORE_MAXINIT;
789
790 if (su->su_badlen >= MAXWLEN)
791 su->su_badlen = MAXWLEN - 1; // just in case
792 vim_strncpy(su->su_badword, su->su_badptr, su->su_badlen);
793 (void)spell_casefold(su->su_badptr, su->su_badlen,
794 su->su_fbadword, MAXWLEN);
795 // TODO: make this work if the case-folded text is longer than the original
796 // text. Currently an illegal byte causes wrong pointer computations.
797 su->su_fbadword[su->su_badlen] = NUL;
798
799 // get caps flags for bad word
800 su->su_badflags = badword_captype(su->su_badptr,
801 su->su_badptr + su->su_badlen);
802 if (need_cap)
803 su->su_badflags |= WF_ONECAP;
804
805 // Find the default language for sound folding. We simply use the first
806 // one in 'spelllang' that supports sound folding. That's good for when
807 // using multiple files for one language, it's not that bad when mixing
808 // languages (e.g., "pl,en").
809 for (i = 0; i < curbuf->b_s.b_langp.ga_len; ++i)
810 {
811 lp = LANGP_ENTRY(curbuf->b_s.b_langp, i);
812 if (lp->lp_sallang != NULL)
813 {
814 su->su_sallang = lp->lp_sallang;
815 break;
816 }
817 }
818
819 // Soundfold the bad word with the default sound folding, so that we don't
820 // have to do this many times.
821 if (su->su_sallang != NULL)
822 spell_soundfold(su->su_sallang, su->su_fbadword, TRUE,
823 su->su_sal_badword);
824
825 // If the word is not capitalised and spell_check() doesn't consider the
826 // word to be bad then it might need to be capitalised. Add a suggestion
827 // for that.
828 c = PTR2CHAR(su->su_badptr);
829 if (!SPELL_ISUPPER(c) && attr == HLF_COUNT)
830 {
831 make_case_word(su->su_badword, buf, WF_ONECAP);
832 add_suggestion(su, &su->su_ga, buf, su->su_badlen, SCORE_ICASE,
833 0, TRUE, su->su_sallang, FALSE);
834 }
835
836 // Ban the bad word itself. It may appear in another region.
837 if (banbadword)
838 add_banned(su, su->su_badword);
839
840 // Make a copy of 'spellsuggest', because the expression may change it.
841 sps_copy = vim_strsave(p_sps);
842 if (sps_copy == NULL)
843 return;
844
845 // Loop over the items in 'spellsuggest'.
846 for (p = sps_copy; *p != NUL; )
847 {
848 copy_option_part(&p, buf, MAXPATHL, ",");
849
850 if (STRNCMP(buf, "expr:", 5) == 0)
851 {
852#ifdef FEAT_EVAL
853 // Evaluate an expression. Skip this when called recursively,
854 // when using spellsuggest() in the expression.
855 if (!expr_busy)
856 {
857 expr_busy = TRUE;
858 spell_suggest_expr(su, buf + 5);
859 expr_busy = FALSE;
860 }
861#endif
862 }
863 else if (STRNCMP(buf, "file:", 5) == 0)
864 // Use list of suggestions in a file.
865 spell_suggest_file(su, buf + 5);
866 else
867 {
868 // Use internal method.
869 spell_suggest_intern(su, interactive);
870 if (sps_flags & SPS_DOUBLE)
871 do_combine = TRUE;
872 }
873 }
874
875 vim_free(sps_copy);
876
877 if (do_combine)
878 // Combine the two list of suggestions. This must be done last,
879 // because sorting changes the order again.
880 score_combine(su);
881}
882
883#ifdef FEAT_EVAL
884/*
885 * Find suggestions by evaluating expression "expr".
886 */
887 static void
888spell_suggest_expr(suginfo_T *su, char_u *expr)
889{
890 list_T *list;
891 listitem_T *li;
892 int score;
893 char_u *p;
894
895 // The work is split up in a few parts to avoid having to export
896 // suginfo_T.
897 // First evaluate the expression and get the resulting list.
898 list = eval_spell_expr(su->su_badword, expr);
899 if (list != NULL)
900 {
901 // Loop over the items in the list.
Bram Moolenaaraeea7212020-04-02 18:50:46 +0200902 FOR_ALL_LIST_ITEMS(list, li)
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200903 if (li->li_tv.v_type == VAR_LIST)
904 {
905 // Get the word and the score from the items.
906 score = get_spellword(li->li_tv.vval.v_list, &p);
907 if (score >= 0 && score <= su->su_maxscore)
908 add_suggestion(su, &su->su_ga, p, su->su_badlen,
909 score, 0, TRUE, su->su_sallang, FALSE);
910 }
911 list_unref(list);
912 }
913
914 // Remove bogus suggestions, sort and truncate at "maxcount".
915 check_suggestions(su, &su->su_ga);
916 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
917}
918#endif
919
920/*
921 * Find suggestions in file "fname". Used for "file:" in 'spellsuggest'.
922 */
923 static void
924spell_suggest_file(suginfo_T *su, char_u *fname)
925{
926 FILE *fd;
927 char_u line[MAXWLEN * 2];
928 char_u *p;
929 int len;
930 char_u cword[MAXWLEN];
931
932 // Open the file.
933 fd = mch_fopen((char *)fname, "r");
934 if (fd == NULL)
935 {
936 semsg(_(e_notopen), fname);
937 return;
938 }
939
940 // Read it line by line.
941 while (!vim_fgets(line, MAXWLEN * 2, fd) && !got_int)
942 {
943 line_breakcheck();
944
945 p = vim_strchr(line, '/');
946 if (p == NULL)
947 continue; // No Tab found, just skip the line.
948 *p++ = NUL;
949 if (STRICMP(su->su_badword, line) == 0)
950 {
951 // Match! Isolate the good word, until CR or NL.
952 for (len = 0; p[len] >= ' '; ++len)
953 ;
954 p[len] = NUL;
955
956 // If the suggestion doesn't have specific case duplicate the case
957 // of the bad word.
958 if (captype(p, NULL) == 0)
959 {
960 make_case_word(p, cword, su->su_badflags);
961 p = cword;
962 }
963
964 add_suggestion(su, &su->su_ga, p, su->su_badlen,
965 SCORE_FILE, 0, TRUE, su->su_sallang, FALSE);
966 }
967 }
968
969 fclose(fd);
970
971 // Remove bogus suggestions, sort and truncate at "maxcount".
972 check_suggestions(su, &su->su_ga);
973 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
974}
975
976/*
977 * Find suggestions for the internal method indicated by "sps_flags".
978 */
979 static void
980spell_suggest_intern(suginfo_T *su, int interactive)
981{
982 // Load the .sug file(s) that are available and not done yet.
983 suggest_load_files();
984
985 // 1. Try special cases, such as repeating a word: "the the" -> "the".
986 //
987 // Set a maximum score to limit the combination of operations that is
988 // tried.
989 suggest_try_special(su);
990
991 // 2. Try inserting/deleting/swapping/changing a letter, use REP entries
992 // from the .aff file and inserting a space (split the word).
993 suggest_try_change(su);
994
995 // For the resulting top-scorers compute the sound-a-like score.
996 if (sps_flags & SPS_DOUBLE)
997 score_comp_sal(su);
998
999 // 3. Try finding sound-a-like words.
1000 if ((sps_flags & SPS_FAST) == 0)
1001 {
1002 if (sps_flags & SPS_BEST)
1003 // Adjust the word score for the suggestions found so far for how
Bram Moolenaar32aa1022019-11-02 22:54:41 +01001004 // they sound like.
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001005 rescore_suggestions(su);
1006
1007 // While going through the soundfold tree "su_maxscore" is the score
1008 // for the soundfold word, limits the changes that are being tried,
1009 // and "su_sfmaxscore" the rescored score, which is set by
1010 // cleanup_suggestions().
1011 // First find words with a small edit distance, because this is much
1012 // faster and often already finds the top-N suggestions. If we didn't
1013 // find many suggestions try again with a higher edit distance.
1014 // "sl_sounddone" is used to avoid doing the same word twice.
1015 suggest_try_soundalike_prep();
1016 su->su_maxscore = SCORE_SFMAX1;
1017 su->su_sfmaxscore = SCORE_MAXINIT * 3;
1018 suggest_try_soundalike(su);
1019 if (su->su_ga.ga_len < SUG_CLEAN_COUNT(su))
1020 {
1021 // We didn't find enough matches, try again, allowing more
1022 // changes to the soundfold word.
1023 su->su_maxscore = SCORE_SFMAX2;
1024 suggest_try_soundalike(su);
1025 if (su->su_ga.ga_len < SUG_CLEAN_COUNT(su))
1026 {
1027 // Still didn't find enough matches, try again, allowing even
1028 // more changes to the soundfold word.
1029 su->su_maxscore = SCORE_SFMAX3;
1030 suggest_try_soundalike(su);
1031 }
1032 }
1033 su->su_maxscore = su->su_sfmaxscore;
1034 suggest_try_soundalike_finish();
1035 }
1036
1037 // When CTRL-C was hit while searching do show the results. Only clear
1038 // got_int when using a command, not for spellsuggest().
1039 ui_breakcheck();
1040 if (interactive && got_int)
1041 {
1042 (void)vgetc();
1043 got_int = FALSE;
1044 }
1045
1046 if ((sps_flags & SPS_DOUBLE) == 0 && su->su_ga.ga_len != 0)
1047 {
1048 if (sps_flags & SPS_BEST)
1049 // Adjust the word score for how it sounds like.
1050 rescore_suggestions(su);
1051
1052 // Remove bogus suggestions, sort and truncate at "maxcount".
1053 check_suggestions(su, &su->su_ga);
1054 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
1055 }
1056}
1057
1058/*
1059 * Free the info put in "*su" by spell_find_suggest().
1060 */
1061 static void
1062spell_find_cleanup(suginfo_T *su)
1063{
1064 int i;
1065
1066 // Free the suggestions.
1067 for (i = 0; i < su->su_ga.ga_len; ++i)
1068 vim_free(SUG(su->su_ga, i).st_word);
1069 ga_clear(&su->su_ga);
1070 for (i = 0; i < su->su_sga.ga_len; ++i)
1071 vim_free(SUG(su->su_sga, i).st_word);
1072 ga_clear(&su->su_sga);
1073
1074 // Free the banned words.
1075 hash_clear_all(&su->su_banned, 0);
1076}
1077
1078/*
1079 * Try finding suggestions by recognizing specific situations.
1080 */
1081 static void
1082suggest_try_special(suginfo_T *su)
1083{
1084 char_u *p;
1085 size_t len;
1086 int c;
1087 char_u word[MAXWLEN];
1088
1089 // Recognize a word that is repeated: "the the".
1090 p = skiptowhite(su->su_fbadword);
1091 len = p - su->su_fbadword;
1092 p = skipwhite(p);
1093 if (STRLEN(p) == len && STRNCMP(su->su_fbadword, p, len) == 0)
1094 {
1095 // Include badflags: if the badword is onecap or allcap
1096 // use that for the goodword too: "The the" -> "The".
1097 c = su->su_fbadword[len];
1098 su->su_fbadword[len] = NUL;
1099 make_case_word(su->su_fbadword, word, su->su_badflags);
1100 su->su_fbadword[len] = c;
1101
1102 // Give a soundalike score of 0, compute the score as if deleting one
1103 // character.
1104 add_suggestion(su, &su->su_ga, word, su->su_badlen,
1105 RESCORE(SCORE_REP, 0), 0, TRUE, su->su_sallang, FALSE);
1106 }
1107}
1108
1109/*
1110 * Change the 0 to 1 to measure how much time is spent in each state.
1111 * Output is dumped in "suggestprof".
1112 */
1113#if 0
1114# define SUGGEST_PROFILE
1115proftime_T current;
1116proftime_T total;
1117proftime_T times[STATE_FINAL + 1];
1118long counts[STATE_FINAL + 1];
1119
1120 static void
1121prof_init(void)
1122{
1123 for (int i = 0; i <= STATE_FINAL; ++i)
1124 {
1125 profile_zero(&times[i]);
1126 counts[i] = 0;
1127 }
1128 profile_start(&current);
1129 profile_start(&total);
1130}
1131
1132// call before changing state
1133 static void
1134prof_store(state_T state)
1135{
1136 profile_end(&current);
1137 profile_add(&times[state], &current);
1138 ++counts[state];
1139 profile_start(&current);
1140}
1141# define PROF_STORE(state) prof_store(state);
1142
1143 static void
1144prof_report(char *name)
1145{
1146 FILE *fd = fopen("suggestprof", "a");
1147
1148 profile_end(&total);
1149 fprintf(fd, "-----------------------\n");
1150 fprintf(fd, "%s: %s\n", name, profile_msg(&total));
1151 for (int i = 0; i <= STATE_FINAL; ++i)
1152 fprintf(fd, "%d: %s (%ld)\n", i, profile_msg(&times[i]), counts[i]);
1153 fclose(fd);
1154}
1155#else
1156# define PROF_STORE(state)
1157#endif
1158
1159/*
1160 * Try finding suggestions by adding/removing/swapping letters.
1161 */
1162 static void
1163suggest_try_change(suginfo_T *su)
1164{
1165 char_u fword[MAXWLEN]; // copy of the bad word, case-folded
1166 int n;
1167 char_u *p;
1168 int lpi;
1169 langp_T *lp;
1170
1171 // We make a copy of the case-folded bad word, so that we can modify it
1172 // to find matches (esp. REP items). Append some more text, changing
1173 // chars after the bad word may help.
1174 STRCPY(fword, su->su_fbadword);
1175 n = (int)STRLEN(fword);
1176 p = su->su_badptr + su->su_badlen;
1177 (void)spell_casefold(p, (int)STRLEN(p), fword + n, MAXWLEN - n);
1178
1179 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
1180 {
1181 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
1182
1183 // If reloading a spell file fails it's still in the list but
1184 // everything has been cleared.
1185 if (lp->lp_slang->sl_fbyts == NULL)
1186 continue;
1187
1188 // Try it for this language. Will add possible suggestions.
1189#ifdef SUGGEST_PROFILE
1190 prof_init();
1191#endif
1192 suggest_trie_walk(su, lp, fword, FALSE);
1193#ifdef SUGGEST_PROFILE
1194 prof_report("try_change");
1195#endif
1196 }
1197}
1198
1199// Check the maximum score, if we go over it we won't try this change.
1200#define TRY_DEEPER(su, stack, depth, add) \
1201 (stack[depth].ts_score + (add) < su->su_maxscore)
1202
1203/*
1204 * Try finding suggestions by adding/removing/swapping letters.
1205 *
1206 * This uses a state machine. At each node in the tree we try various
1207 * operations. When trying if an operation works "depth" is increased and the
1208 * stack[] is used to store info. This allows combinations, thus insert one
1209 * character, replace one and delete another. The number of changes is
1210 * limited by su->su_maxscore.
1211 *
1212 * After implementing this I noticed an article by Kemal Oflazer that
1213 * describes something similar: "Error-tolerant Finite State Recognition with
1214 * Applications to Morphological Analysis and Spelling Correction" (1996).
1215 * The implementation in the article is simplified and requires a stack of
1216 * unknown depth. The implementation here only needs a stack depth equal to
1217 * the length of the word.
1218 *
1219 * This is also used for the sound-folded word, "soundfold" is TRUE then.
1220 * The mechanism is the same, but we find a match with a sound-folded word
1221 * that comes from one or more original words. Each of these words may be
1222 * added, this is done by add_sound_suggest().
1223 * Don't use:
1224 * the prefix tree or the keep-case tree
1225 * "su->su_badlen"
1226 * anything to do with upper and lower case
1227 * anything to do with word or non-word characters ("spell_iswordp()")
1228 * banned words
1229 * word flags (rare, region, compounding)
1230 * word splitting for now
1231 * "similar_chars()"
1232 * use "slang->sl_repsal" instead of "lp->lp_replang->sl_rep"
1233 */
1234 static void
1235suggest_trie_walk(
1236 suginfo_T *su,
1237 langp_T *lp,
1238 char_u *fword,
1239 int soundfold)
1240{
1241 char_u tword[MAXWLEN]; // good word collected so far
1242 trystate_T stack[MAXWLEN];
1243 char_u preword[MAXWLEN * 3]; // word found with proper case;
1244 // concatenation of prefix compound
1245 // words and split word. NUL terminated
1246 // when going deeper but not when coming
1247 // back.
1248 char_u compflags[MAXWLEN]; // compound flags, one for each word
1249 trystate_T *sp;
1250 int newscore;
1251 int score;
1252 char_u *byts, *fbyts, *pbyts;
1253 idx_T *idxs, *fidxs, *pidxs;
1254 int depth;
1255 int c, c2, c3;
1256 int n = 0;
1257 int flags;
1258 garray_T *gap;
1259 idx_T arridx;
1260 int len;
1261 char_u *p;
1262 fromto_T *ftp;
1263 int fl = 0, tl;
1264 int repextra = 0; // extra bytes in fword[] from REP item
1265 slang_T *slang = lp->lp_slang;
1266 int fword_ends;
1267 int goodword_ends;
1268#ifdef DEBUG_TRIEWALK
1269 // Stores the name of the change made at each level.
1270 char_u changename[MAXWLEN][80];
1271#endif
1272 int breakcheckcount = 1000;
1273 int compound_ok;
1274
1275 // Go through the whole case-fold tree, try changes at each node.
1276 // "tword[]" contains the word collected from nodes in the tree.
1277 // "fword[]" the word we are trying to match with (initially the bad
1278 // word).
1279 depth = 0;
1280 sp = &stack[0];
Bram Moolenaara80faa82020-04-12 19:37:17 +02001281 CLEAR_POINTER(sp);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001282 sp->ts_curi = 1;
1283
1284 if (soundfold)
1285 {
1286 // Going through the soundfold tree.
1287 byts = fbyts = slang->sl_sbyts;
1288 idxs = fidxs = slang->sl_sidxs;
1289 pbyts = NULL;
1290 pidxs = NULL;
1291 sp->ts_prefixdepth = PFD_NOPREFIX;
1292 sp->ts_state = STATE_START;
1293 }
1294 else
1295 {
1296 // When there are postponed prefixes we need to use these first. At
1297 // the end of the prefix we continue in the case-fold tree.
1298 fbyts = slang->sl_fbyts;
1299 fidxs = slang->sl_fidxs;
1300 pbyts = slang->sl_pbyts;
1301 pidxs = slang->sl_pidxs;
1302 if (pbyts != NULL)
1303 {
1304 byts = pbyts;
1305 idxs = pidxs;
1306 sp->ts_prefixdepth = PFD_PREFIXTREE;
1307 sp->ts_state = STATE_NOPREFIX; // try without prefix first
1308 }
1309 else
1310 {
1311 byts = fbyts;
1312 idxs = fidxs;
1313 sp->ts_prefixdepth = PFD_NOPREFIX;
1314 sp->ts_state = STATE_START;
1315 }
1316 }
1317
1318 // Loop to find all suggestions. At each round we either:
1319 // - For the current state try one operation, advance "ts_curi",
1320 // increase "depth".
1321 // - When a state is done go to the next, set "ts_state".
1322 // - When all states are tried decrease "depth".
1323 while (depth >= 0 && !got_int)
1324 {
1325 sp = &stack[depth];
1326 switch (sp->ts_state)
1327 {
1328 case STATE_START:
1329 case STATE_NOPREFIX:
1330 // Start of node: Deal with NUL bytes, which means
1331 // tword[] may end here.
1332 arridx = sp->ts_arridx; // current node in the tree
1333 len = byts[arridx]; // bytes in this node
1334 arridx += sp->ts_curi; // index of current byte
1335
1336 if (sp->ts_prefixdepth == PFD_PREFIXTREE)
1337 {
1338 // Skip over the NUL bytes, we use them later.
1339 for (n = 0; n < len && byts[arridx + n] == 0; ++n)
1340 ;
1341 sp->ts_curi += n;
1342
1343 // Always past NUL bytes now.
1344 n = (int)sp->ts_state;
1345 PROF_STORE(sp->ts_state)
1346 sp->ts_state = STATE_ENDNUL;
1347 sp->ts_save_badflags = su->su_badflags;
1348
1349 // At end of a prefix or at start of prefixtree: check for
1350 // following word.
1351 if (byts[arridx] == 0 || n == (int)STATE_NOPREFIX)
1352 {
1353 // Set su->su_badflags to the caps type at this position.
1354 // Use the caps type until here for the prefix itself.
1355 if (has_mbyte)
1356 n = nofold_len(fword, sp->ts_fidx, su->su_badptr);
1357 else
1358 n = sp->ts_fidx;
1359 flags = badword_captype(su->su_badptr, su->su_badptr + n);
1360 su->su_badflags = badword_captype(su->su_badptr + n,
1361 su->su_badptr + su->su_badlen);
1362#ifdef DEBUG_TRIEWALK
1363 sprintf(changename[depth], "prefix");
1364#endif
1365 go_deeper(stack, depth, 0);
1366 ++depth;
1367 sp = &stack[depth];
1368 sp->ts_prefixdepth = depth - 1;
1369 byts = fbyts;
1370 idxs = fidxs;
1371 sp->ts_arridx = 0;
1372
1373 // Move the prefix to preword[] with the right case
1374 // and make find_keepcap_word() works.
1375 tword[sp->ts_twordlen] = NUL;
1376 make_case_word(tword + sp->ts_splitoff,
1377 preword + sp->ts_prewordlen, flags);
1378 sp->ts_prewordlen = (char_u)STRLEN(preword);
1379 sp->ts_splitoff = sp->ts_twordlen;
1380 }
1381 break;
1382 }
1383
1384 if (sp->ts_curi > len || byts[arridx] != 0)
1385 {
1386 // Past bytes in node and/or past NUL bytes.
1387 PROF_STORE(sp->ts_state)
1388 sp->ts_state = STATE_ENDNUL;
1389 sp->ts_save_badflags = su->su_badflags;
1390 break;
1391 }
1392
1393 // End of word in tree.
1394 ++sp->ts_curi; // eat one NUL byte
1395
1396 flags = (int)idxs[arridx];
1397
1398 // Skip words with the NOSUGGEST flag.
1399 if (flags & WF_NOSUGGEST)
1400 break;
1401
1402 fword_ends = (fword[sp->ts_fidx] == NUL
1403 || (soundfold
1404 ? VIM_ISWHITE(fword[sp->ts_fidx])
1405 : !spell_iswordp(fword + sp->ts_fidx, curwin)));
1406 tword[sp->ts_twordlen] = NUL;
1407
1408 if (sp->ts_prefixdepth <= PFD_NOTSPECIAL
Bram Moolenaar11b66002020-07-01 13:15:24 +02001409 && (sp->ts_flags & TSF_PREFIXOK) == 0
1410 && pbyts != NULL)
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001411 {
1412 // There was a prefix before the word. Check that the prefix
1413 // can be used with this word.
1414 // Count the length of the NULs in the prefix. If there are
1415 // none this must be the first try without a prefix.
1416 n = stack[sp->ts_prefixdepth].ts_arridx;
1417 len = pbyts[n++];
1418 for (c = 0; c < len && pbyts[n + c] == 0; ++c)
1419 ;
1420 if (c > 0)
1421 {
1422 c = valid_word_prefix(c, n, flags,
1423 tword + sp->ts_splitoff, slang, FALSE);
1424 if (c == 0)
1425 break;
1426
1427 // Use the WF_RARE flag for a rare prefix.
1428 if (c & WF_RAREPFX)
1429 flags |= WF_RARE;
1430
1431 // Tricky: when checking for both prefix and compounding
1432 // we run into the prefix flag first.
1433 // Remember that it's OK, so that we accept the prefix
1434 // when arriving at a compound flag.
1435 sp->ts_flags |= TSF_PREFIXOK;
1436 }
1437 }
1438
1439 // Check NEEDCOMPOUND: can't use word without compounding. Do try
1440 // appending another compound word below.
1441 if (sp->ts_complen == sp->ts_compsplit && fword_ends
1442 && (flags & WF_NEEDCOMP))
1443 goodword_ends = FALSE;
1444 else
1445 goodword_ends = TRUE;
1446
1447 p = NULL;
1448 compound_ok = TRUE;
1449 if (sp->ts_complen > sp->ts_compsplit)
1450 {
1451 if (slang->sl_nobreak)
1452 {
1453 // There was a word before this word. When there was no
1454 // change in this word (it was correct) add the first word
1455 // as a suggestion. If this word was corrected too, we
1456 // need to check if a correct word follows.
1457 if (sp->ts_fidx - sp->ts_splitfidx
1458 == sp->ts_twordlen - sp->ts_splitoff
1459 && STRNCMP(fword + sp->ts_splitfidx,
1460 tword + sp->ts_splitoff,
1461 sp->ts_fidx - sp->ts_splitfidx) == 0)
1462 {
1463 preword[sp->ts_prewordlen] = NUL;
1464 newscore = score_wordcount_adj(slang, sp->ts_score,
1465 preword + sp->ts_prewordlen,
1466 sp->ts_prewordlen > 0);
1467 // Add the suggestion if the score isn't too bad.
1468 if (newscore <= su->su_maxscore)
1469 add_suggestion(su, &su->su_ga, preword,
1470 sp->ts_splitfidx - repextra,
1471 newscore, 0, FALSE,
1472 lp->lp_sallang, FALSE);
1473 break;
1474 }
1475 }
1476 else
1477 {
1478 // There was a compound word before this word. If this
1479 // word does not support compounding then give up
1480 // (splitting is tried for the word without compound
1481 // flag).
1482 if (((unsigned)flags >> 24) == 0
1483 || sp->ts_twordlen - sp->ts_splitoff
1484 < slang->sl_compminlen)
1485 break;
1486 // For multi-byte chars check character length against
1487 // COMPOUNDMIN.
1488 if (has_mbyte
1489 && slang->sl_compminlen > 0
1490 && mb_charlen(tword + sp->ts_splitoff)
1491 < slang->sl_compminlen)
1492 break;
1493
1494 compflags[sp->ts_complen] = ((unsigned)flags >> 24);
1495 compflags[sp->ts_complen + 1] = NUL;
1496 vim_strncpy(preword + sp->ts_prewordlen,
1497 tword + sp->ts_splitoff,
1498 sp->ts_twordlen - sp->ts_splitoff);
1499
1500 // Verify CHECKCOMPOUNDPATTERN rules.
1501 if (match_checkcompoundpattern(preword, sp->ts_prewordlen,
1502 &slang->sl_comppat))
1503 compound_ok = FALSE;
1504
1505 if (compound_ok)
1506 {
1507 p = preword;
1508 while (*skiptowhite(p) != NUL)
1509 p = skipwhite(skiptowhite(p));
1510 if (fword_ends && !can_compound(slang, p,
1511 compflags + sp->ts_compsplit))
1512 // Compound is not allowed. But it may still be
1513 // possible if we add another (short) word.
1514 compound_ok = FALSE;
1515 }
1516
1517 // Get pointer to last char of previous word.
1518 p = preword + sp->ts_prewordlen;
1519 MB_PTR_BACK(preword, p);
1520 }
1521 }
1522
1523 // Form the word with proper case in preword.
1524 // If there is a word from a previous split, append.
1525 // For the soundfold tree don't change the case, simply append.
1526 if (soundfold)
1527 STRCPY(preword + sp->ts_prewordlen, tword + sp->ts_splitoff);
1528 else if (flags & WF_KEEPCAP)
1529 // Must find the word in the keep-case tree.
1530 find_keepcap_word(slang, tword + sp->ts_splitoff,
1531 preword + sp->ts_prewordlen);
1532 else
1533 {
1534 // Include badflags: If the badword is onecap or allcap
1535 // use that for the goodword too. But if the badword is
1536 // allcap and it's only one char long use onecap.
1537 c = su->su_badflags;
1538 if ((c & WF_ALLCAP)
1539 && su->su_badlen == (*mb_ptr2len)(su->su_badptr))
1540 c = WF_ONECAP;
1541 c |= flags;
1542
1543 // When appending a compound word after a word character don't
1544 // use Onecap.
1545 if (p != NULL && spell_iswordp_nmw(p, curwin))
1546 c &= ~WF_ONECAP;
1547 make_case_word(tword + sp->ts_splitoff,
1548 preword + sp->ts_prewordlen, c);
1549 }
1550
1551 if (!soundfold)
1552 {
1553 // Don't use a banned word. It may appear again as a good
1554 // word, thus remember it.
1555 if (flags & WF_BANNED)
1556 {
1557 add_banned(su, preword + sp->ts_prewordlen);
1558 break;
1559 }
1560 if ((sp->ts_complen == sp->ts_compsplit
1561 && WAS_BANNED(su, preword + sp->ts_prewordlen))
1562 || WAS_BANNED(su, preword))
1563 {
1564 if (slang->sl_compprog == NULL)
1565 break;
1566 // the word so far was banned but we may try compounding
1567 goodword_ends = FALSE;
1568 }
1569 }
1570
1571 newscore = 0;
1572 if (!soundfold) // soundfold words don't have flags
1573 {
1574 if ((flags & WF_REGION)
1575 && (((unsigned)flags >> 16) & lp->lp_region) == 0)
1576 newscore += SCORE_REGION;
1577 if (flags & WF_RARE)
1578 newscore += SCORE_RARE;
1579
1580 if (!spell_valid_case(su->su_badflags,
1581 captype(preword + sp->ts_prewordlen, NULL)))
1582 newscore += SCORE_ICASE;
1583 }
1584
1585 // TODO: how about splitting in the soundfold tree?
1586 if (fword_ends
1587 && goodword_ends
1588 && sp->ts_fidx >= sp->ts_fidxtry
1589 && compound_ok)
1590 {
1591 // The badword also ends: add suggestions.
1592#ifdef DEBUG_TRIEWALK
1593 if (soundfold && STRCMP(preword, "smwrd") == 0)
1594 {
1595 int j;
1596
1597 // print the stack of changes that brought us here
1598 smsg("------ %s -------", fword);
1599 for (j = 0; j < depth; ++j)
1600 smsg("%s", changename[j]);
1601 }
1602#endif
1603 if (soundfold)
1604 {
1605 // For soundfolded words we need to find the original
1606 // words, the edit distance and then add them.
1607 add_sound_suggest(su, preword, sp->ts_score, lp);
1608 }
1609 else if (sp->ts_fidx > 0)
1610 {
1611 // Give a penalty when changing non-word char to word
1612 // char, e.g., "thes," -> "these".
1613 p = fword + sp->ts_fidx;
1614 MB_PTR_BACK(fword, p);
1615 if (!spell_iswordp(p, curwin))
1616 {
1617 p = preword + STRLEN(preword);
1618 MB_PTR_BACK(preword, p);
1619 if (spell_iswordp(p, curwin))
1620 newscore += SCORE_NONWORD;
1621 }
1622
1623 // Give a bonus to words seen before.
1624 score = score_wordcount_adj(slang,
1625 sp->ts_score + newscore,
1626 preword + sp->ts_prewordlen,
1627 sp->ts_prewordlen > 0);
1628
1629 // Add the suggestion if the score isn't too bad.
1630 if (score <= su->su_maxscore)
1631 {
1632 add_suggestion(su, &su->su_ga, preword,
1633 sp->ts_fidx - repextra,
1634 score, 0, FALSE, lp->lp_sallang, FALSE);
1635
1636 if (su->su_badflags & WF_MIXCAP)
1637 {
1638 // We really don't know if the word should be
1639 // upper or lower case, add both.
1640 c = captype(preword, NULL);
1641 if (c == 0 || c == WF_ALLCAP)
1642 {
1643 make_case_word(tword + sp->ts_splitoff,
1644 preword + sp->ts_prewordlen,
1645 c == 0 ? WF_ALLCAP : 0);
1646
1647 add_suggestion(su, &su->su_ga, preword,
1648 sp->ts_fidx - repextra,
1649 score + SCORE_ICASE, 0, FALSE,
1650 lp->lp_sallang, FALSE);
1651 }
1652 }
1653 }
1654 }
1655 }
1656
1657 // Try word split and/or compounding.
1658 if ((sp->ts_fidx >= sp->ts_fidxtry || fword_ends)
1659 // Don't split halfway a character.
1660 && (!has_mbyte || sp->ts_tcharlen == 0))
1661 {
1662 int try_compound;
1663 int try_split;
1664
1665 // If past the end of the bad word don't try a split.
1666 // Otherwise try changing the next word. E.g., find
1667 // suggestions for "the the" where the second "the" is
1668 // different. It's done like a split.
1669 // TODO: word split for soundfold words
1670 try_split = (sp->ts_fidx - repextra < su->su_badlen)
1671 && !soundfold;
1672
1673 // Get here in several situations:
1674 // 1. The word in the tree ends:
1675 // If the word allows compounding try that. Otherwise try
1676 // a split by inserting a space. For both check that a
1677 // valid words starts at fword[sp->ts_fidx].
1678 // For NOBREAK do like compounding to be able to check if
1679 // the next word is valid.
1680 // 2. The badword does end, but it was due to a change (e.g.,
1681 // a swap). No need to split, but do check that the
1682 // following word is valid.
1683 // 3. The badword and the word in the tree end. It may still
1684 // be possible to compound another (short) word.
1685 try_compound = FALSE;
1686 if (!soundfold
1687 && !slang->sl_nocompoundsugs
1688 && slang->sl_compprog != NULL
1689 && ((unsigned)flags >> 24) != 0
1690 && sp->ts_twordlen - sp->ts_splitoff
1691 >= slang->sl_compminlen
1692 && (!has_mbyte
1693 || slang->sl_compminlen == 0
1694 || mb_charlen(tword + sp->ts_splitoff)
1695 >= slang->sl_compminlen)
1696 && (slang->sl_compsylmax < MAXWLEN
1697 || sp->ts_complen + 1 - sp->ts_compsplit
1698 < slang->sl_compmax)
1699 && (can_be_compound(sp, slang,
1700 compflags, ((unsigned)flags >> 24))))
1701
1702 {
1703 try_compound = TRUE;
1704 compflags[sp->ts_complen] = ((unsigned)flags >> 24);
1705 compflags[sp->ts_complen + 1] = NUL;
1706 }
1707
1708 // For NOBREAK we never try splitting, it won't make any word
1709 // valid.
1710 if (slang->sl_nobreak && !slang->sl_nocompoundsugs)
1711 try_compound = TRUE;
1712
1713 // If we could add a compound word, and it's also possible to
1714 // split at this point, do the split first and set
1715 // TSF_DIDSPLIT to avoid doing it again.
1716 else if (!fword_ends
1717 && try_compound
1718 && (sp->ts_flags & TSF_DIDSPLIT) == 0)
1719 {
1720 try_compound = FALSE;
1721 sp->ts_flags |= TSF_DIDSPLIT;
1722 --sp->ts_curi; // do the same NUL again
1723 compflags[sp->ts_complen] = NUL;
1724 }
1725 else
1726 sp->ts_flags &= ~TSF_DIDSPLIT;
1727
1728 if (try_split || try_compound)
1729 {
1730 if (!try_compound && (!fword_ends || !goodword_ends))
1731 {
1732 // If we're going to split need to check that the
1733 // words so far are valid for compounding. If there
1734 // is only one word it must not have the NEEDCOMPOUND
1735 // flag.
1736 if (sp->ts_complen == sp->ts_compsplit
1737 && (flags & WF_NEEDCOMP))
1738 break;
1739 p = preword;
1740 while (*skiptowhite(p) != NUL)
1741 p = skipwhite(skiptowhite(p));
1742 if (sp->ts_complen > sp->ts_compsplit
1743 && !can_compound(slang, p,
1744 compflags + sp->ts_compsplit))
1745 break;
1746
1747 if (slang->sl_nosplitsugs)
1748 newscore += SCORE_SPLIT_NO;
1749 else
1750 newscore += SCORE_SPLIT;
1751
1752 // Give a bonus to words seen before.
1753 newscore = score_wordcount_adj(slang, newscore,
1754 preword + sp->ts_prewordlen, TRUE);
1755 }
1756
1757 if (TRY_DEEPER(su, stack, depth, newscore))
1758 {
1759 go_deeper(stack, depth, newscore);
1760#ifdef DEBUG_TRIEWALK
1761 if (!try_compound && !fword_ends)
1762 sprintf(changename[depth], "%.*s-%s: split",
1763 sp->ts_twordlen, tword, fword + sp->ts_fidx);
1764 else
1765 sprintf(changename[depth], "%.*s-%s: compound",
1766 sp->ts_twordlen, tword, fword + sp->ts_fidx);
1767#endif
1768 // Save things to be restored at STATE_SPLITUNDO.
1769 sp->ts_save_badflags = su->su_badflags;
1770 PROF_STORE(sp->ts_state)
1771 sp->ts_state = STATE_SPLITUNDO;
1772
1773 ++depth;
1774 sp = &stack[depth];
1775
1776 // Append a space to preword when splitting.
1777 if (!try_compound && !fword_ends)
1778 STRCAT(preword, " ");
1779 sp->ts_prewordlen = (char_u)STRLEN(preword);
1780 sp->ts_splitoff = sp->ts_twordlen;
1781 sp->ts_splitfidx = sp->ts_fidx;
1782
1783 // If the badword has a non-word character at this
1784 // position skip it. That means replacing the
1785 // non-word character with a space. Always skip a
1786 // character when the word ends. But only when the
1787 // good word can end.
1788 if (((!try_compound && !spell_iswordp_nmw(fword
1789 + sp->ts_fidx,
1790 curwin))
1791 || fword_ends)
1792 && fword[sp->ts_fidx] != NUL
1793 && goodword_ends)
1794 {
1795 int l;
1796
Bram Moolenaar1614a142019-10-06 22:00:13 +02001797 l = mb_ptr2len(fword + sp->ts_fidx);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001798 if (fword_ends)
1799 {
1800 // Copy the skipped character to preword.
1801 mch_memmove(preword + sp->ts_prewordlen,
1802 fword + sp->ts_fidx, l);
1803 sp->ts_prewordlen += l;
1804 preword[sp->ts_prewordlen] = NUL;
1805 }
1806 else
1807 sp->ts_score -= SCORE_SPLIT - SCORE_SUBST;
1808 sp->ts_fidx += l;
1809 }
1810
1811 // When compounding include compound flag in
1812 // compflags[] (already set above). When splitting we
1813 // may start compounding over again.
1814 if (try_compound)
1815 ++sp->ts_complen;
1816 else
1817 sp->ts_compsplit = sp->ts_complen;
1818 sp->ts_prefixdepth = PFD_NOPREFIX;
1819
1820 // set su->su_badflags to the caps type at this
1821 // position
1822 if (has_mbyte)
1823 n = nofold_len(fword, sp->ts_fidx, su->su_badptr);
1824 else
1825 n = sp->ts_fidx;
1826 su->su_badflags = badword_captype(su->su_badptr + n,
1827 su->su_badptr + su->su_badlen);
1828
1829 // Restart at top of the tree.
1830 sp->ts_arridx = 0;
1831
1832 // If there are postponed prefixes, try these too.
1833 if (pbyts != NULL)
1834 {
1835 byts = pbyts;
1836 idxs = pidxs;
1837 sp->ts_prefixdepth = PFD_PREFIXTREE;
1838 PROF_STORE(sp->ts_state)
1839 sp->ts_state = STATE_NOPREFIX;
1840 }
1841 }
1842 }
1843 }
1844 break;
1845
1846 case STATE_SPLITUNDO:
1847 // Undo the changes done for word split or compound word.
1848 su->su_badflags = sp->ts_save_badflags;
1849
1850 // Continue looking for NUL bytes.
1851 PROF_STORE(sp->ts_state)
1852 sp->ts_state = STATE_START;
1853
1854 // In case we went into the prefix tree.
1855 byts = fbyts;
1856 idxs = fidxs;
1857 break;
1858
1859 case STATE_ENDNUL:
1860 // Past the NUL bytes in the node.
1861 su->su_badflags = sp->ts_save_badflags;
1862 if (fword[sp->ts_fidx] == NUL && sp->ts_tcharlen == 0)
1863 {
1864 // The badword ends, can't use STATE_PLAIN.
1865 PROF_STORE(sp->ts_state)
1866 sp->ts_state = STATE_DEL;
1867 break;
1868 }
1869 PROF_STORE(sp->ts_state)
1870 sp->ts_state = STATE_PLAIN;
1871 // FALLTHROUGH
1872
1873 case STATE_PLAIN:
1874 // Go over all possible bytes at this node, add each to tword[]
1875 // and use child node. "ts_curi" is the index.
1876 arridx = sp->ts_arridx;
1877 if (sp->ts_curi > byts[arridx])
1878 {
1879 // Done all bytes at this node, do next state. When still at
1880 // already changed bytes skip the other tricks.
1881 PROF_STORE(sp->ts_state)
1882 if (sp->ts_fidx >= sp->ts_fidxtry)
1883 sp->ts_state = STATE_DEL;
1884 else
1885 sp->ts_state = STATE_FINAL;
1886 }
1887 else
1888 {
1889 arridx += sp->ts_curi++;
1890 c = byts[arridx];
1891
1892 // Normal byte, go one level deeper. If it's not equal to the
1893 // byte in the bad word adjust the score. But don't even try
1894 // when the byte was already changed. And don't try when we
1895 // just deleted this byte, accepting it is always cheaper than
1896 // delete + substitute.
1897 if (c == fword[sp->ts_fidx]
1898 || (sp->ts_tcharlen > 0 && sp->ts_isdiff != DIFF_NONE))
1899 newscore = 0;
1900 else
1901 newscore = SCORE_SUBST;
1902 if ((newscore == 0
1903 || (sp->ts_fidx >= sp->ts_fidxtry
1904 && ((sp->ts_flags & TSF_DIDDEL) == 0
1905 || c != fword[sp->ts_delidx])))
1906 && TRY_DEEPER(su, stack, depth, newscore))
1907 {
1908 go_deeper(stack, depth, newscore);
1909#ifdef DEBUG_TRIEWALK
1910 if (newscore > 0)
1911 sprintf(changename[depth], "%.*s-%s: subst %c to %c",
1912 sp->ts_twordlen, tword, fword + sp->ts_fidx,
1913 fword[sp->ts_fidx], c);
1914 else
1915 sprintf(changename[depth], "%.*s-%s: accept %c",
1916 sp->ts_twordlen, tword, fword + sp->ts_fidx,
1917 fword[sp->ts_fidx]);
1918#endif
1919 ++depth;
1920 sp = &stack[depth];
1921 ++sp->ts_fidx;
1922 tword[sp->ts_twordlen++] = c;
1923 sp->ts_arridx = idxs[arridx];
1924 if (newscore == SCORE_SUBST)
1925 sp->ts_isdiff = DIFF_YES;
1926 if (has_mbyte)
1927 {
1928 // Multi-byte characters are a bit complicated to
1929 // handle: They differ when any of the bytes differ
1930 // and then their length may also differ.
1931 if (sp->ts_tcharlen == 0)
1932 {
1933 // First byte.
1934 sp->ts_tcharidx = 0;
1935 sp->ts_tcharlen = MB_BYTE2LEN(c);
1936 sp->ts_fcharstart = sp->ts_fidx - 1;
1937 sp->ts_isdiff = (newscore != 0)
1938 ? DIFF_YES : DIFF_NONE;
1939 }
1940 else if (sp->ts_isdiff == DIFF_INSERT)
1941 // When inserting trail bytes don't advance in the
1942 // bad word.
1943 --sp->ts_fidx;
1944 if (++sp->ts_tcharidx == sp->ts_tcharlen)
1945 {
1946 // Last byte of character.
1947 if (sp->ts_isdiff == DIFF_YES)
1948 {
1949 // Correct ts_fidx for the byte length of the
1950 // character (we didn't check that before).
1951 sp->ts_fidx = sp->ts_fcharstart
Bram Moolenaar1614a142019-10-06 22:00:13 +02001952 + mb_ptr2len(
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001953 fword + sp->ts_fcharstart);
1954 // For changing a composing character adjust
1955 // the score from SCORE_SUBST to
1956 // SCORE_SUBCOMP.
1957 if (enc_utf8
1958 && utf_iscomposing(
1959 utf_ptr2char(tword
1960 + sp->ts_twordlen
1961 - sp->ts_tcharlen))
1962 && utf_iscomposing(
1963 utf_ptr2char(fword
1964 + sp->ts_fcharstart)))
1965 sp->ts_score -=
1966 SCORE_SUBST - SCORE_SUBCOMP;
1967
1968 // For a similar character adjust score from
1969 // SCORE_SUBST to SCORE_SIMILAR.
1970 else if (!soundfold
1971 && slang->sl_has_map
1972 && similar_chars(slang,
1973 mb_ptr2char(tword
1974 + sp->ts_twordlen
1975 - sp->ts_tcharlen),
1976 mb_ptr2char(fword
1977 + sp->ts_fcharstart)))
1978 sp->ts_score -=
1979 SCORE_SUBST - SCORE_SIMILAR;
1980 }
1981 else if (sp->ts_isdiff == DIFF_INSERT
1982 && sp->ts_twordlen > sp->ts_tcharlen)
1983 {
1984 p = tword + sp->ts_twordlen - sp->ts_tcharlen;
1985 c = mb_ptr2char(p);
1986 if (enc_utf8 && utf_iscomposing(c))
1987 {
1988 // Inserting a composing char doesn't
1989 // count that much.
1990 sp->ts_score -= SCORE_INS - SCORE_INSCOMP;
1991 }
1992 else
1993 {
1994 // If the previous character was the same,
1995 // thus doubling a character, give a bonus
1996 // to the score. Also for the soundfold
1997 // tree (might seem illogical but does
1998 // give better scores).
1999 MB_PTR_BACK(tword, p);
2000 if (c == mb_ptr2char(p))
2001 sp->ts_score -= SCORE_INS
2002 - SCORE_INSDUP;
2003 }
2004 }
2005
2006 // Starting a new char, reset the length.
2007 sp->ts_tcharlen = 0;
2008 }
2009 }
2010 else
2011 {
2012 // If we found a similar char adjust the score.
2013 // We do this after calling go_deeper() because
2014 // it's slow.
2015 if (newscore != 0
2016 && !soundfold
2017 && slang->sl_has_map
2018 && similar_chars(slang,
2019 c, fword[sp->ts_fidx - 1]))
2020 sp->ts_score -= SCORE_SUBST - SCORE_SIMILAR;
2021 }
2022 }
2023 }
2024 break;
2025
2026 case STATE_DEL:
2027 // When past the first byte of a multi-byte char don't try
2028 // delete/insert/swap a character.
2029 if (has_mbyte && sp->ts_tcharlen > 0)
2030 {
2031 PROF_STORE(sp->ts_state)
2032 sp->ts_state = STATE_FINAL;
2033 break;
2034 }
2035 // Try skipping one character in the bad word (delete it).
2036 PROF_STORE(sp->ts_state)
2037 sp->ts_state = STATE_INS_PREP;
2038 sp->ts_curi = 1;
2039 if (soundfold && sp->ts_fidx == 0 && fword[sp->ts_fidx] == '*')
2040 // Deleting a vowel at the start of a word counts less, see
2041 // soundalike_score().
2042 newscore = 2 * SCORE_DEL / 3;
2043 else
2044 newscore = SCORE_DEL;
2045 if (fword[sp->ts_fidx] != NUL
2046 && TRY_DEEPER(su, stack, depth, newscore))
2047 {
2048 go_deeper(stack, depth, newscore);
2049#ifdef DEBUG_TRIEWALK
2050 sprintf(changename[depth], "%.*s-%s: delete %c",
2051 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2052 fword[sp->ts_fidx]);
2053#endif
2054 ++depth;
2055
2056 // Remember what character we deleted, so that we can avoid
2057 // inserting it again.
2058 stack[depth].ts_flags |= TSF_DIDDEL;
2059 stack[depth].ts_delidx = sp->ts_fidx;
2060
2061 // Advance over the character in fword[]. Give a bonus to the
2062 // score if the same character is following "nn" -> "n". It's
2063 // a bit illogical for soundfold tree but it does give better
2064 // results.
2065 if (has_mbyte)
2066 {
2067 c = mb_ptr2char(fword + sp->ts_fidx);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002068 stack[depth].ts_fidx += mb_ptr2len(fword + sp->ts_fidx);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002069 if (enc_utf8 && utf_iscomposing(c))
2070 stack[depth].ts_score -= SCORE_DEL - SCORE_DELCOMP;
2071 else if (c == mb_ptr2char(fword + stack[depth].ts_fidx))
2072 stack[depth].ts_score -= SCORE_DEL - SCORE_DELDUP;
2073 }
2074 else
2075 {
2076 ++stack[depth].ts_fidx;
2077 if (fword[sp->ts_fidx] == fword[sp->ts_fidx + 1])
2078 stack[depth].ts_score -= SCORE_DEL - SCORE_DELDUP;
2079 }
2080 break;
2081 }
2082 // FALLTHROUGH
2083
2084 case STATE_INS_PREP:
2085 if (sp->ts_flags & TSF_DIDDEL)
2086 {
2087 // If we just deleted a byte then inserting won't make sense,
2088 // a substitute is always cheaper.
2089 PROF_STORE(sp->ts_state)
2090 sp->ts_state = STATE_SWAP;
2091 break;
2092 }
2093
2094 // skip over NUL bytes
2095 n = sp->ts_arridx;
2096 for (;;)
2097 {
2098 if (sp->ts_curi > byts[n])
2099 {
2100 // Only NUL bytes at this node, go to next state.
2101 PROF_STORE(sp->ts_state)
2102 sp->ts_state = STATE_SWAP;
2103 break;
2104 }
2105 if (byts[n + sp->ts_curi] != NUL)
2106 {
2107 // Found a byte to insert.
2108 PROF_STORE(sp->ts_state)
2109 sp->ts_state = STATE_INS;
2110 break;
2111 }
2112 ++sp->ts_curi;
2113 }
2114 break;
2115
2116 // FALLTHROUGH
2117
2118 case STATE_INS:
2119 // Insert one byte. Repeat this for each possible byte at this
2120 // node.
2121 n = sp->ts_arridx;
2122 if (sp->ts_curi > byts[n])
2123 {
2124 // Done all bytes at this node, go to next state.
2125 PROF_STORE(sp->ts_state)
2126 sp->ts_state = STATE_SWAP;
2127 break;
2128 }
2129
2130 // Do one more byte at this node, but:
2131 // - Skip NUL bytes.
2132 // - Skip the byte if it's equal to the byte in the word,
2133 // accepting that byte is always better.
2134 n += sp->ts_curi++;
2135 c = byts[n];
2136 if (soundfold && sp->ts_twordlen == 0 && c == '*')
2137 // Inserting a vowel at the start of a word counts less,
2138 // see soundalike_score().
2139 newscore = 2 * SCORE_INS / 3;
2140 else
2141 newscore = SCORE_INS;
2142 if (c != fword[sp->ts_fidx]
2143 && TRY_DEEPER(su, stack, depth, newscore))
2144 {
2145 go_deeper(stack, depth, newscore);
2146#ifdef DEBUG_TRIEWALK
2147 sprintf(changename[depth], "%.*s-%s: insert %c",
2148 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2149 c);
2150#endif
2151 ++depth;
2152 sp = &stack[depth];
2153 tword[sp->ts_twordlen++] = c;
2154 sp->ts_arridx = idxs[n];
2155 if (has_mbyte)
2156 {
2157 fl = MB_BYTE2LEN(c);
2158 if (fl > 1)
2159 {
2160 // There are following bytes for the same character.
2161 // We must find all bytes before trying
2162 // delete/insert/swap/etc.
2163 sp->ts_tcharlen = fl;
2164 sp->ts_tcharidx = 1;
2165 sp->ts_isdiff = DIFF_INSERT;
2166 }
2167 }
2168 else
2169 fl = 1;
2170 if (fl == 1)
2171 {
2172 // If the previous character was the same, thus doubling a
2173 // character, give a bonus to the score. Also for
2174 // soundfold words (illogical but does give a better
2175 // score).
2176 if (sp->ts_twordlen >= 2
2177 && tword[sp->ts_twordlen - 2] == c)
2178 sp->ts_score -= SCORE_INS - SCORE_INSDUP;
2179 }
2180 }
2181 break;
2182
2183 case STATE_SWAP:
2184 // Swap two bytes in the bad word: "12" -> "21".
2185 // We change "fword" here, it's changed back afterwards at
2186 // STATE_UNSWAP.
2187 p = fword + sp->ts_fidx;
2188 c = *p;
2189 if (c == NUL)
2190 {
2191 // End of word, can't swap or replace.
2192 PROF_STORE(sp->ts_state)
2193 sp->ts_state = STATE_FINAL;
2194 break;
2195 }
2196
2197 // Don't swap if the first character is not a word character.
2198 // SWAP3 etc. also don't make sense then.
2199 if (!soundfold && !spell_iswordp(p, curwin))
2200 {
2201 PROF_STORE(sp->ts_state)
2202 sp->ts_state = STATE_REP_INI;
2203 break;
2204 }
2205
2206 if (has_mbyte)
2207 {
2208 n = MB_CPTR2LEN(p);
2209 c = mb_ptr2char(p);
2210 if (p[n] == NUL)
2211 c2 = NUL;
2212 else if (!soundfold && !spell_iswordp(p + n, curwin))
2213 c2 = c; // don't swap non-word char
2214 else
2215 c2 = mb_ptr2char(p + n);
2216 }
2217 else
2218 {
2219 if (p[1] == NUL)
2220 c2 = NUL;
2221 else if (!soundfold && !spell_iswordp(p + 1, curwin))
2222 c2 = c; // don't swap non-word char
2223 else
2224 c2 = p[1];
2225 }
2226
2227 // When the second character is NUL we can't swap.
2228 if (c2 == NUL)
2229 {
2230 PROF_STORE(sp->ts_state)
2231 sp->ts_state = STATE_REP_INI;
2232 break;
2233 }
2234
2235 // When characters are identical, swap won't do anything.
2236 // Also get here if the second char is not a word character.
2237 if (c == c2)
2238 {
2239 PROF_STORE(sp->ts_state)
2240 sp->ts_state = STATE_SWAP3;
2241 break;
2242 }
2243 if (c2 != NUL && TRY_DEEPER(su, stack, depth, SCORE_SWAP))
2244 {
2245 go_deeper(stack, depth, SCORE_SWAP);
2246#ifdef DEBUG_TRIEWALK
2247 sprintf(changename[depth], "%.*s-%s: swap %c and %c",
2248 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2249 c, c2);
2250#endif
2251 PROF_STORE(sp->ts_state)
2252 sp->ts_state = STATE_UNSWAP;
2253 ++depth;
2254 if (has_mbyte)
2255 {
2256 fl = mb_char2len(c2);
2257 mch_memmove(p, p + n, fl);
2258 mb_char2bytes(c, p + fl);
2259 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl;
2260 }
2261 else
2262 {
2263 p[0] = c2;
2264 p[1] = c;
2265 stack[depth].ts_fidxtry = sp->ts_fidx + 2;
2266 }
2267 }
2268 else
2269 {
2270 // If this swap doesn't work then SWAP3 won't either.
2271 PROF_STORE(sp->ts_state)
2272 sp->ts_state = STATE_REP_INI;
2273 }
2274 break;
2275
2276 case STATE_UNSWAP:
2277 // Undo the STATE_SWAP swap: "21" -> "12".
2278 p = fword + sp->ts_fidx;
2279 if (has_mbyte)
2280 {
Bram Moolenaar1614a142019-10-06 22:00:13 +02002281 n = mb_ptr2len(p);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002282 c = mb_ptr2char(p + n);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002283 mch_memmove(p + mb_ptr2len(p + n), p, n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002284 mb_char2bytes(c, p);
2285 }
2286 else
2287 {
2288 c = *p;
2289 *p = p[1];
2290 p[1] = c;
2291 }
2292 // FALLTHROUGH
2293
2294 case STATE_SWAP3:
2295 // Swap two bytes, skipping one: "123" -> "321". We change
2296 // "fword" here, it's changed back afterwards at STATE_UNSWAP3.
2297 p = fword + sp->ts_fidx;
2298 if (has_mbyte)
2299 {
2300 n = MB_CPTR2LEN(p);
2301 c = mb_ptr2char(p);
2302 fl = MB_CPTR2LEN(p + n);
2303 c2 = mb_ptr2char(p + n);
2304 if (!soundfold && !spell_iswordp(p + n + fl, curwin))
2305 c3 = c; // don't swap non-word char
2306 else
2307 c3 = mb_ptr2char(p + n + fl);
2308 }
2309 else
2310 {
2311 c = *p;
2312 c2 = p[1];
2313 if (!soundfold && !spell_iswordp(p + 2, curwin))
2314 c3 = c; // don't swap non-word char
2315 else
2316 c3 = p[2];
2317 }
2318
2319 // When characters are identical: "121" then SWAP3 result is
2320 // identical, ROT3L result is same as SWAP: "211", ROT3L result is
2321 // same as SWAP on next char: "112". Thus skip all swapping.
2322 // Also skip when c3 is NUL.
2323 // Also get here when the third character is not a word character.
2324 // Second character may any char: "a.b" -> "b.a"
2325 if (c == c3 || c3 == NUL)
2326 {
2327 PROF_STORE(sp->ts_state)
2328 sp->ts_state = STATE_REP_INI;
2329 break;
2330 }
2331 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3))
2332 {
2333 go_deeper(stack, depth, SCORE_SWAP3);
2334#ifdef DEBUG_TRIEWALK
2335 sprintf(changename[depth], "%.*s-%s: swap3 %c and %c",
2336 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2337 c, c3);
2338#endif
2339 PROF_STORE(sp->ts_state)
2340 sp->ts_state = STATE_UNSWAP3;
2341 ++depth;
2342 if (has_mbyte)
2343 {
2344 tl = mb_char2len(c3);
2345 mch_memmove(p, p + n + fl, tl);
2346 mb_char2bytes(c2, p + tl);
2347 mb_char2bytes(c, p + fl + tl);
2348 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl + tl;
2349 }
2350 else
2351 {
2352 p[0] = p[2];
2353 p[2] = c;
2354 stack[depth].ts_fidxtry = sp->ts_fidx + 3;
2355 }
2356 }
2357 else
2358 {
2359 PROF_STORE(sp->ts_state)
2360 sp->ts_state = STATE_REP_INI;
2361 }
2362 break;
2363
2364 case STATE_UNSWAP3:
2365 // Undo STATE_SWAP3: "321" -> "123"
2366 p = fword + sp->ts_fidx;
2367 if (has_mbyte)
2368 {
Bram Moolenaar1614a142019-10-06 22:00:13 +02002369 n = mb_ptr2len(p);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002370 c2 = mb_ptr2char(p + n);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002371 fl = mb_ptr2len(p + n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002372 c = mb_ptr2char(p + n + fl);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002373 tl = mb_ptr2len(p + n + fl);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002374 mch_memmove(p + fl + tl, p, n);
2375 mb_char2bytes(c, p);
2376 mb_char2bytes(c2, p + tl);
2377 p = p + tl;
2378 }
2379 else
2380 {
2381 c = *p;
2382 *p = p[2];
2383 p[2] = c;
2384 ++p;
2385 }
2386
2387 if (!soundfold && !spell_iswordp(p, curwin))
2388 {
2389 // Middle char is not a word char, skip the rotate. First and
2390 // third char were already checked at swap and swap3.
2391 PROF_STORE(sp->ts_state)
2392 sp->ts_state = STATE_REP_INI;
2393 break;
2394 }
2395
2396 // Rotate three characters left: "123" -> "231". We change
2397 // "fword" here, it's changed back afterwards at STATE_UNROT3L.
2398 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3))
2399 {
2400 go_deeper(stack, depth, SCORE_SWAP3);
2401#ifdef DEBUG_TRIEWALK
2402 p = fword + sp->ts_fidx;
2403 sprintf(changename[depth], "%.*s-%s: rotate left %c%c%c",
2404 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2405 p[0], p[1], p[2]);
2406#endif
2407 PROF_STORE(sp->ts_state)
2408 sp->ts_state = STATE_UNROT3L;
2409 ++depth;
2410 p = fword + sp->ts_fidx;
2411 if (has_mbyte)
2412 {
2413 n = MB_CPTR2LEN(p);
2414 c = mb_ptr2char(p);
2415 fl = MB_CPTR2LEN(p + n);
2416 fl += MB_CPTR2LEN(p + n + fl);
2417 mch_memmove(p, p + n, fl);
2418 mb_char2bytes(c, p + fl);
2419 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl;
2420 }
2421 else
2422 {
2423 c = *p;
2424 *p = p[1];
2425 p[1] = p[2];
2426 p[2] = c;
2427 stack[depth].ts_fidxtry = sp->ts_fidx + 3;
2428 }
2429 }
2430 else
2431 {
2432 PROF_STORE(sp->ts_state)
2433 sp->ts_state = STATE_REP_INI;
2434 }
2435 break;
2436
2437 case STATE_UNROT3L:
2438 // Undo ROT3L: "231" -> "123"
2439 p = fword + sp->ts_fidx;
2440 if (has_mbyte)
2441 {
Bram Moolenaar1614a142019-10-06 22:00:13 +02002442 n = mb_ptr2len(p);
2443 n += mb_ptr2len(p + n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002444 c = mb_ptr2char(p + n);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002445 tl = mb_ptr2len(p + n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002446 mch_memmove(p + tl, p, n);
2447 mb_char2bytes(c, p);
2448 }
2449 else
2450 {
2451 c = p[2];
2452 p[2] = p[1];
2453 p[1] = *p;
2454 *p = c;
2455 }
2456
2457 // Rotate three bytes right: "123" -> "312". We change "fword"
2458 // here, it's changed back afterwards at STATE_UNROT3R.
2459 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3))
2460 {
2461 go_deeper(stack, depth, SCORE_SWAP3);
2462#ifdef DEBUG_TRIEWALK
2463 p = fword + sp->ts_fidx;
2464 sprintf(changename[depth], "%.*s-%s: rotate right %c%c%c",
2465 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2466 p[0], p[1], p[2]);
2467#endif
2468 PROF_STORE(sp->ts_state)
2469 sp->ts_state = STATE_UNROT3R;
2470 ++depth;
2471 p = fword + sp->ts_fidx;
2472 if (has_mbyte)
2473 {
2474 n = MB_CPTR2LEN(p);
2475 n += MB_CPTR2LEN(p + n);
2476 c = mb_ptr2char(p + n);
2477 tl = MB_CPTR2LEN(p + n);
2478 mch_memmove(p + tl, p, n);
2479 mb_char2bytes(c, p);
2480 stack[depth].ts_fidxtry = sp->ts_fidx + n + tl;
2481 }
2482 else
2483 {
2484 c = p[2];
2485 p[2] = p[1];
2486 p[1] = *p;
2487 *p = c;
2488 stack[depth].ts_fidxtry = sp->ts_fidx + 3;
2489 }
2490 }
2491 else
2492 {
2493 PROF_STORE(sp->ts_state)
2494 sp->ts_state = STATE_REP_INI;
2495 }
2496 break;
2497
2498 case STATE_UNROT3R:
2499 // Undo ROT3R: "312" -> "123"
2500 p = fword + sp->ts_fidx;
2501 if (has_mbyte)
2502 {
2503 c = mb_ptr2char(p);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002504 tl = mb_ptr2len(p);
2505 n = mb_ptr2len(p + tl);
2506 n += mb_ptr2len(p + tl + n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002507 mch_memmove(p, p + tl, n);
2508 mb_char2bytes(c, p + n);
2509 }
2510 else
2511 {
2512 c = *p;
2513 *p = p[1];
2514 p[1] = p[2];
2515 p[2] = c;
2516 }
2517 // FALLTHROUGH
2518
2519 case STATE_REP_INI:
2520 // Check if matching with REP items from the .aff file would work.
2521 // Quickly skip if:
2522 // - there are no REP items and we are not in the soundfold trie
2523 // - the score is going to be too high anyway
2524 // - already applied a REP item or swapped here
2525 if ((lp->lp_replang == NULL && !soundfold)
2526 || sp->ts_score + SCORE_REP >= su->su_maxscore
2527 || sp->ts_fidx < sp->ts_fidxtry)
2528 {
2529 PROF_STORE(sp->ts_state)
2530 sp->ts_state = STATE_FINAL;
2531 break;
2532 }
2533
2534 // Use the first byte to quickly find the first entry that may
2535 // match. If the index is -1 there is none.
2536 if (soundfold)
2537 sp->ts_curi = slang->sl_repsal_first[fword[sp->ts_fidx]];
2538 else
2539 sp->ts_curi = lp->lp_replang->sl_rep_first[fword[sp->ts_fidx]];
2540
2541 if (sp->ts_curi < 0)
2542 {
2543 PROF_STORE(sp->ts_state)
2544 sp->ts_state = STATE_FINAL;
2545 break;
2546 }
2547
2548 PROF_STORE(sp->ts_state)
2549 sp->ts_state = STATE_REP;
2550 // FALLTHROUGH
2551
2552 case STATE_REP:
2553 // Try matching with REP items from the .aff file. For each match
2554 // replace the characters and check if the resulting word is
2555 // valid.
2556 p = fword + sp->ts_fidx;
2557
2558 if (soundfold)
2559 gap = &slang->sl_repsal;
2560 else
2561 gap = &lp->lp_replang->sl_rep;
2562 while (sp->ts_curi < gap->ga_len)
2563 {
2564 ftp = (fromto_T *)gap->ga_data + sp->ts_curi++;
2565 if (*ftp->ft_from != *p)
2566 {
2567 // past possible matching entries
2568 sp->ts_curi = gap->ga_len;
2569 break;
2570 }
2571 if (STRNCMP(ftp->ft_from, p, STRLEN(ftp->ft_from)) == 0
2572 && TRY_DEEPER(su, stack, depth, SCORE_REP))
2573 {
2574 go_deeper(stack, depth, SCORE_REP);
2575#ifdef DEBUG_TRIEWALK
2576 sprintf(changename[depth], "%.*s-%s: replace %s with %s",
2577 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2578 ftp->ft_from, ftp->ft_to);
2579#endif
2580 // Need to undo this afterwards.
2581 PROF_STORE(sp->ts_state)
2582 sp->ts_state = STATE_REP_UNDO;
2583
2584 // Change the "from" to the "to" string.
2585 ++depth;
2586 fl = (int)STRLEN(ftp->ft_from);
2587 tl = (int)STRLEN(ftp->ft_to);
2588 if (fl != tl)
2589 {
2590 STRMOVE(p + tl, p + fl);
2591 repextra += tl - fl;
2592 }
2593 mch_memmove(p, ftp->ft_to, tl);
2594 stack[depth].ts_fidxtry = sp->ts_fidx + tl;
2595 stack[depth].ts_tcharlen = 0;
2596 break;
2597 }
2598 }
2599
2600 if (sp->ts_curi >= gap->ga_len && sp->ts_state == STATE_REP)
2601 {
2602 // No (more) matches.
2603 PROF_STORE(sp->ts_state)
2604 sp->ts_state = STATE_FINAL;
2605 }
2606
2607 break;
2608
2609 case STATE_REP_UNDO:
2610 // Undo a REP replacement and continue with the next one.
2611 if (soundfold)
2612 gap = &slang->sl_repsal;
2613 else
2614 gap = &lp->lp_replang->sl_rep;
2615 ftp = (fromto_T *)gap->ga_data + sp->ts_curi - 1;
2616 fl = (int)STRLEN(ftp->ft_from);
2617 tl = (int)STRLEN(ftp->ft_to);
2618 p = fword + sp->ts_fidx;
2619 if (fl != tl)
2620 {
2621 STRMOVE(p + fl, p + tl);
2622 repextra -= tl - fl;
2623 }
2624 mch_memmove(p, ftp->ft_from, fl);
2625 PROF_STORE(sp->ts_state)
2626 sp->ts_state = STATE_REP;
2627 break;
2628
2629 default:
2630 // Did all possible states at this level, go up one level.
2631 --depth;
2632
2633 if (depth >= 0 && stack[depth].ts_prefixdepth == PFD_PREFIXTREE)
2634 {
2635 // Continue in or go back to the prefix tree.
2636 byts = pbyts;
2637 idxs = pidxs;
2638 }
2639
2640 // Don't check for CTRL-C too often, it takes time.
2641 if (--breakcheckcount == 0)
2642 {
2643 ui_breakcheck();
2644 breakcheckcount = 1000;
2645 }
2646 }
2647 }
2648}
2649
2650
2651/*
2652 * Go one level deeper in the tree.
2653 */
2654 static void
2655go_deeper(trystate_T *stack, int depth, int score_add)
2656{
2657 stack[depth + 1] = stack[depth];
2658 stack[depth + 1].ts_state = STATE_START;
2659 stack[depth + 1].ts_score = stack[depth].ts_score + score_add;
2660 stack[depth + 1].ts_curi = 1; // start just after length byte
2661 stack[depth + 1].ts_flags = 0;
2662}
2663
2664/*
2665 * "fword" is a good word with case folded. Find the matching keep-case
2666 * words and put it in "kword".
2667 * Theoretically there could be several keep-case words that result in the
2668 * same case-folded word, but we only find one...
2669 */
2670 static void
2671find_keepcap_word(slang_T *slang, char_u *fword, char_u *kword)
2672{
2673 char_u uword[MAXWLEN]; // "fword" in upper-case
2674 int depth;
2675 idx_T tryidx;
2676
2677 // The following arrays are used at each depth in the tree.
2678 idx_T arridx[MAXWLEN];
2679 int round[MAXWLEN];
2680 int fwordidx[MAXWLEN];
2681 int uwordidx[MAXWLEN];
2682 int kwordlen[MAXWLEN];
2683
2684 int flen, ulen;
2685 int l;
2686 int len;
2687 int c;
2688 idx_T lo, hi, m;
2689 char_u *p;
2690 char_u *byts = slang->sl_kbyts; // array with bytes of the words
2691 idx_T *idxs = slang->sl_kidxs; // array with indexes
2692
2693 if (byts == NULL)
2694 {
2695 // array is empty: "cannot happen"
2696 *kword = NUL;
2697 return;
2698 }
2699
2700 // Make an all-cap version of "fword".
2701 allcap_copy(fword, uword);
2702
2703 // Each character needs to be tried both case-folded and upper-case.
2704 // All this gets very complicated if we keep in mind that changing case
2705 // may change the byte length of a multi-byte character...
2706 depth = 0;
2707 arridx[0] = 0;
2708 round[0] = 0;
2709 fwordidx[0] = 0;
2710 uwordidx[0] = 0;
2711 kwordlen[0] = 0;
2712 while (depth >= 0)
2713 {
2714 if (fword[fwordidx[depth]] == NUL)
2715 {
2716 // We are at the end of "fword". If the tree allows a word to end
2717 // here we have found a match.
2718 if (byts[arridx[depth] + 1] == 0)
2719 {
2720 kword[kwordlen[depth]] = NUL;
2721 return;
2722 }
2723
2724 // kword is getting too long, continue one level up
2725 --depth;
2726 }
2727 else if (++round[depth] > 2)
2728 {
2729 // tried both fold-case and upper-case character, continue one
2730 // level up
2731 --depth;
2732 }
2733 else
2734 {
2735 // round[depth] == 1: Try using the folded-case character.
2736 // round[depth] == 2: Try using the upper-case character.
2737 if (has_mbyte)
2738 {
2739 flen = MB_CPTR2LEN(fword + fwordidx[depth]);
2740 ulen = MB_CPTR2LEN(uword + uwordidx[depth]);
2741 }
2742 else
2743 ulen = flen = 1;
2744 if (round[depth] == 1)
2745 {
2746 p = fword + fwordidx[depth];
2747 l = flen;
2748 }
2749 else
2750 {
2751 p = uword + uwordidx[depth];
2752 l = ulen;
2753 }
2754
2755 for (tryidx = arridx[depth]; l > 0; --l)
2756 {
2757 // Perform a binary search in the list of accepted bytes.
2758 len = byts[tryidx++];
2759 c = *p++;
2760 lo = tryidx;
2761 hi = tryidx + len - 1;
2762 while (lo < hi)
2763 {
2764 m = (lo + hi) / 2;
2765 if (byts[m] > c)
2766 hi = m - 1;
2767 else if (byts[m] < c)
2768 lo = m + 1;
2769 else
2770 {
2771 lo = hi = m;
2772 break;
2773 }
2774 }
2775
2776 // Stop if there is no matching byte.
2777 if (hi < lo || byts[lo] != c)
2778 break;
2779
2780 // Continue at the child (if there is one).
2781 tryidx = idxs[lo];
2782 }
2783
2784 if (l == 0)
2785 {
2786 // Found the matching char. Copy it to "kword" and go a
2787 // level deeper.
2788 if (round[depth] == 1)
2789 {
2790 STRNCPY(kword + kwordlen[depth], fword + fwordidx[depth],
2791 flen);
2792 kwordlen[depth + 1] = kwordlen[depth] + flen;
2793 }
2794 else
2795 {
2796 STRNCPY(kword + kwordlen[depth], uword + uwordidx[depth],
2797 ulen);
2798 kwordlen[depth + 1] = kwordlen[depth] + ulen;
2799 }
2800 fwordidx[depth + 1] = fwordidx[depth] + flen;
2801 uwordidx[depth + 1] = uwordidx[depth] + ulen;
2802
2803 ++depth;
2804 arridx[depth] = tryidx;
2805 round[depth] = 0;
2806 }
2807 }
2808 }
2809
2810 // Didn't find it: "cannot happen".
2811 *kword = NUL;
2812}
2813
2814/*
2815 * Compute the sound-a-like score for suggestions in su->su_ga and add them to
2816 * su->su_sga.
2817 */
2818 static void
2819score_comp_sal(suginfo_T *su)
2820{
2821 langp_T *lp;
2822 char_u badsound[MAXWLEN];
2823 int i;
2824 suggest_T *stp;
2825 suggest_T *sstp;
2826 int score;
2827 int lpi;
2828
2829 if (ga_grow(&su->su_sga, su->su_ga.ga_len) == FAIL)
2830 return;
2831
2832 // Use the sound-folding of the first language that supports it.
2833 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
2834 {
2835 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
2836 if (lp->lp_slang->sl_sal.ga_len > 0)
2837 {
2838 // soundfold the bad word
2839 spell_soundfold(lp->lp_slang, su->su_fbadword, TRUE, badsound);
2840
2841 for (i = 0; i < su->su_ga.ga_len; ++i)
2842 {
2843 stp = &SUG(su->su_ga, i);
2844
2845 // Case-fold the suggested word, sound-fold it and compute the
2846 // sound-a-like score.
2847 score = stp_sal_score(stp, su, lp->lp_slang, badsound);
2848 if (score < SCORE_MAXMAX)
2849 {
2850 // Add the suggestion.
2851 sstp = &SUG(su->su_sga, su->su_sga.ga_len);
2852 sstp->st_word = vim_strsave(stp->st_word);
2853 if (sstp->st_word != NULL)
2854 {
2855 sstp->st_wordlen = stp->st_wordlen;
2856 sstp->st_score = score;
2857 sstp->st_altscore = 0;
2858 sstp->st_orglen = stp->st_orglen;
2859 ++su->su_sga.ga_len;
2860 }
2861 }
2862 }
2863 break;
2864 }
2865 }
2866}
2867
2868/*
2869 * Combine the list of suggestions in su->su_ga and su->su_sga.
2870 * They are entwined.
2871 */
2872 static void
2873score_combine(suginfo_T *su)
2874{
2875 int i;
2876 int j;
2877 garray_T ga;
2878 garray_T *gap;
2879 langp_T *lp;
2880 suggest_T *stp;
2881 char_u *p;
2882 char_u badsound[MAXWLEN];
2883 int round;
2884 int lpi;
2885 slang_T *slang = NULL;
2886
2887 // Add the alternate score to su_ga.
2888 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
2889 {
2890 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
2891 if (lp->lp_slang->sl_sal.ga_len > 0)
2892 {
2893 // soundfold the bad word
2894 slang = lp->lp_slang;
2895 spell_soundfold(slang, su->su_fbadword, TRUE, badsound);
2896
2897 for (i = 0; i < su->su_ga.ga_len; ++i)
2898 {
2899 stp = &SUG(su->su_ga, i);
2900 stp->st_altscore = stp_sal_score(stp, su, slang, badsound);
2901 if (stp->st_altscore == SCORE_MAXMAX)
2902 stp->st_score = (stp->st_score * 3 + SCORE_BIG) / 4;
2903 else
2904 stp->st_score = (stp->st_score * 3
2905 + stp->st_altscore) / 4;
2906 stp->st_salscore = FALSE;
2907 }
2908 break;
2909 }
2910 }
2911
2912 if (slang == NULL) // Using "double" without sound folding.
2913 {
2914 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore,
2915 su->su_maxcount);
2916 return;
2917 }
2918
2919 // Add the alternate score to su_sga.
2920 for (i = 0; i < su->su_sga.ga_len; ++i)
2921 {
2922 stp = &SUG(su->su_sga, i);
2923 stp->st_altscore = spell_edit_score(slang,
2924 su->su_badword, stp->st_word);
2925 if (stp->st_score == SCORE_MAXMAX)
2926 stp->st_score = (SCORE_BIG * 7 + stp->st_altscore) / 8;
2927 else
2928 stp->st_score = (stp->st_score * 7 + stp->st_altscore) / 8;
2929 stp->st_salscore = TRUE;
2930 }
2931
2932 // Remove bad suggestions, sort the suggestions and truncate at "maxcount"
2933 // for both lists.
2934 check_suggestions(su, &su->su_ga);
2935 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
2936 check_suggestions(su, &su->su_sga);
2937 (void)cleanup_suggestions(&su->su_sga, su->su_maxscore, su->su_maxcount);
2938
2939 ga_init2(&ga, (int)sizeof(suginfo_T), 1);
2940 if (ga_grow(&ga, su->su_ga.ga_len + su->su_sga.ga_len) == FAIL)
2941 return;
2942
2943 stp = &SUG(ga, 0);
2944 for (i = 0; i < su->su_ga.ga_len || i < su->su_sga.ga_len; ++i)
2945 {
2946 // round 1: get a suggestion from su_ga
2947 // round 2: get a suggestion from su_sga
2948 for (round = 1; round <= 2; ++round)
2949 {
2950 gap = round == 1 ? &su->su_ga : &su->su_sga;
2951 if (i < gap->ga_len)
2952 {
2953 // Don't add a word if it's already there.
2954 p = SUG(*gap, i).st_word;
2955 for (j = 0; j < ga.ga_len; ++j)
2956 if (STRCMP(stp[j].st_word, p) == 0)
2957 break;
2958 if (j == ga.ga_len)
2959 stp[ga.ga_len++] = SUG(*gap, i);
2960 else
2961 vim_free(p);
2962 }
2963 }
2964 }
2965
2966 ga_clear(&su->su_ga);
2967 ga_clear(&su->su_sga);
2968
2969 // Truncate the list to the number of suggestions that will be displayed.
2970 if (ga.ga_len > su->su_maxcount)
2971 {
2972 for (i = su->su_maxcount; i < ga.ga_len; ++i)
2973 vim_free(stp[i].st_word);
2974 ga.ga_len = su->su_maxcount;
2975 }
2976
2977 su->su_ga = ga;
2978}
2979
2980/*
2981 * For the goodword in "stp" compute the soundalike score compared to the
2982 * badword.
2983 */
2984 static int
2985stp_sal_score(
2986 suggest_T *stp,
2987 suginfo_T *su,
2988 slang_T *slang,
2989 char_u *badsound) // sound-folded badword
2990{
2991 char_u *p;
2992 char_u *pbad;
2993 char_u *pgood;
2994 char_u badsound2[MAXWLEN];
2995 char_u fword[MAXWLEN];
2996 char_u goodsound[MAXWLEN];
2997 char_u goodword[MAXWLEN];
2998 int lendiff;
2999
3000 lendiff = (int)(su->su_badlen - stp->st_orglen);
3001 if (lendiff >= 0)
3002 pbad = badsound;
3003 else
3004 {
3005 // soundfold the bad word with more characters following
3006 (void)spell_casefold(su->su_badptr, stp->st_orglen, fword, MAXWLEN);
3007
3008 // When joining two words the sound often changes a lot. E.g., "t he"
3009 // sounds like "t h" while "the" sounds like "@". Avoid that by
3010 // removing the space. Don't do it when the good word also contains a
3011 // space.
3012 if (VIM_ISWHITE(su->su_badptr[su->su_badlen])
3013 && *skiptowhite(stp->st_word) == NUL)
3014 for (p = fword; *(p = skiptowhite(p)) != NUL; )
3015 STRMOVE(p, p + 1);
3016
3017 spell_soundfold(slang, fword, TRUE, badsound2);
3018 pbad = badsound2;
3019 }
3020
3021 if (lendiff > 0 && stp->st_wordlen + lendiff < MAXWLEN)
3022 {
3023 // Add part of the bad word to the good word, so that we soundfold
3024 // what replaces the bad word.
3025 STRCPY(goodword, stp->st_word);
3026 vim_strncpy(goodword + stp->st_wordlen,
3027 su->su_badptr + su->su_badlen - lendiff, lendiff);
3028 pgood = goodword;
3029 }
3030 else
3031 pgood = stp->st_word;
3032
3033 // Sound-fold the word and compute the score for the difference.
3034 spell_soundfold(slang, pgood, FALSE, goodsound);
3035
3036 return soundalike_score(goodsound, pbad);
3037}
3038
3039// structure used to store soundfolded words that add_sound_suggest() has
3040// handled already.
3041typedef struct
3042{
3043 short sft_score; // lowest score used
3044 char_u sft_word[1]; // soundfolded word, actually longer
3045} sftword_T;
3046
3047static sftword_T dumsft;
3048#define HIKEY2SFT(p) ((sftword_T *)(p - (dumsft.sft_word - (char_u *)&dumsft)))
3049#define HI2SFT(hi) HIKEY2SFT((hi)->hi_key)
3050
3051/*
3052 * Prepare for calling suggest_try_soundalike().
3053 */
3054 static void
3055suggest_try_soundalike_prep(void)
3056{
3057 langp_T *lp;
3058 int lpi;
3059 slang_T *slang;
3060
3061 // Do this for all languages that support sound folding and for which a
3062 // .sug file has been loaded.
3063 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
3064 {
3065 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
3066 slang = lp->lp_slang;
3067 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL)
3068 // prepare the hashtable used by add_sound_suggest()
3069 hash_init(&slang->sl_sounddone);
3070 }
3071}
3072
3073/*
3074 * Find suggestions by comparing the word in a sound-a-like form.
3075 * Note: This doesn't support postponed prefixes.
3076 */
3077 static void
3078suggest_try_soundalike(suginfo_T *su)
3079{
3080 char_u salword[MAXWLEN];
3081 langp_T *lp;
3082 int lpi;
3083 slang_T *slang;
3084
3085 // Do this for all languages that support sound folding and for which a
3086 // .sug file has been loaded.
3087 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
3088 {
3089 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
3090 slang = lp->lp_slang;
3091 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL)
3092 {
3093 // soundfold the bad word
3094 spell_soundfold(slang, su->su_fbadword, TRUE, salword);
3095
3096 // try all kinds of inserts/deletes/swaps/etc.
3097 // TODO: also soundfold the next words, so that we can try joining
3098 // and splitting
3099#ifdef SUGGEST_PROFILE
3100 prof_init();
3101#endif
3102 suggest_trie_walk(su, lp, salword, TRUE);
3103#ifdef SUGGEST_PROFILE
3104 prof_report("soundalike");
3105#endif
3106 }
3107 }
3108}
3109
3110/*
3111 * Finish up after calling suggest_try_soundalike().
3112 */
3113 static void
3114suggest_try_soundalike_finish(void)
3115{
3116 langp_T *lp;
3117 int lpi;
3118 slang_T *slang;
3119 int todo;
3120 hashitem_T *hi;
3121
3122 // Do this for all languages that support sound folding and for which a
3123 // .sug file has been loaded.
3124 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
3125 {
3126 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
3127 slang = lp->lp_slang;
3128 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL)
3129 {
3130 // Free the info about handled words.
3131 todo = (int)slang->sl_sounddone.ht_used;
3132 for (hi = slang->sl_sounddone.ht_array; todo > 0; ++hi)
3133 if (!HASHITEM_EMPTY(hi))
3134 {
3135 vim_free(HI2SFT(hi));
3136 --todo;
3137 }
3138
3139 // Clear the hashtable, it may also be used by another region.
3140 hash_clear(&slang->sl_sounddone);
3141 hash_init(&slang->sl_sounddone);
3142 }
3143 }
3144}
3145
3146/*
3147 * A match with a soundfolded word is found. Add the good word(s) that
3148 * produce this soundfolded word.
3149 */
3150 static void
3151add_sound_suggest(
3152 suginfo_T *su,
3153 char_u *goodword,
3154 int score, // soundfold score
3155 langp_T *lp)
3156{
3157 slang_T *slang = lp->lp_slang; // language for sound folding
3158 int sfwordnr;
3159 char_u *nrline;
3160 int orgnr;
3161 char_u theword[MAXWLEN];
3162 int i;
3163 int wlen;
3164 char_u *byts;
3165 idx_T *idxs;
3166 int n;
3167 int wordcount;
3168 int wc;
3169 int goodscore;
3170 hash_T hash;
3171 hashitem_T *hi;
3172 sftword_T *sft;
3173 int bc, gc;
3174 int limit;
3175
3176 // It's very well possible that the same soundfold word is found several
3177 // times with different scores. Since the following is quite slow only do
3178 // the words that have a better score than before. Use a hashtable to
3179 // remember the words that have been done.
3180 hash = hash_hash(goodword);
3181 hi = hash_lookup(&slang->sl_sounddone, goodword, hash);
3182 if (HASHITEM_EMPTY(hi))
3183 {
3184 sft = alloc(sizeof(sftword_T) + STRLEN(goodword));
3185 if (sft != NULL)
3186 {
3187 sft->sft_score = score;
3188 STRCPY(sft->sft_word, goodword);
3189 hash_add_item(&slang->sl_sounddone, hi, sft->sft_word, hash);
3190 }
3191 }
3192 else
3193 {
3194 sft = HI2SFT(hi);
3195 if (score >= sft->sft_score)
3196 return;
3197 sft->sft_score = score;
3198 }
3199
3200 // Find the word nr in the soundfold tree.
3201 sfwordnr = soundfold_find(slang, goodword);
3202 if (sfwordnr < 0)
3203 {
3204 internal_error("add_sound_suggest()");
3205 return;
3206 }
3207
3208 // go over the list of good words that produce this soundfold word
3209 nrline = ml_get_buf(slang->sl_sugbuf, (linenr_T)(sfwordnr + 1), FALSE);
3210 orgnr = 0;
3211 while (*nrline != NUL)
3212 {
3213 // The wordnr was stored in a minimal nr of bytes as an offset to the
3214 // previous wordnr.
3215 orgnr += bytes2offset(&nrline);
3216
3217 byts = slang->sl_fbyts;
3218 idxs = slang->sl_fidxs;
3219
3220 // Lookup the word "orgnr" one of the two tries.
3221 n = 0;
3222 wordcount = 0;
3223 for (wlen = 0; wlen < MAXWLEN - 3; ++wlen)
3224 {
3225 i = 1;
3226 if (wordcount == orgnr && byts[n + 1] == NUL)
3227 break; // found end of word
3228
3229 if (byts[n + 1] == NUL)
3230 ++wordcount;
3231
3232 // skip over the NUL bytes
3233 for ( ; byts[n + i] == NUL; ++i)
3234 if (i > byts[n]) // safety check
3235 {
3236 STRCPY(theword + wlen, "BAD");
3237 wlen += 3;
3238 goto badword;
3239 }
3240
3241 // One of the siblings must have the word.
3242 for ( ; i < byts[n]; ++i)
3243 {
3244 wc = idxs[idxs[n + i]]; // nr of words under this byte
3245 if (wordcount + wc > orgnr)
3246 break;
3247 wordcount += wc;
3248 }
3249
3250 theword[wlen] = byts[n + i];
3251 n = idxs[n + i];
3252 }
3253badword:
3254 theword[wlen] = NUL;
3255
3256 // Go over the possible flags and regions.
3257 for (; i <= byts[n] && byts[n + i] == NUL; ++i)
3258 {
3259 char_u cword[MAXWLEN];
3260 char_u *p;
3261 int flags = (int)idxs[n + i];
3262
3263 // Skip words with the NOSUGGEST flag
3264 if (flags & WF_NOSUGGEST)
3265 continue;
3266
3267 if (flags & WF_KEEPCAP)
3268 {
3269 // Must find the word in the keep-case tree.
3270 find_keepcap_word(slang, theword, cword);
3271 p = cword;
3272 }
3273 else
3274 {
3275 flags |= su->su_badflags;
3276 if ((flags & WF_CAPMASK) != 0)
3277 {
3278 // Need to fix case according to "flags".
3279 make_case_word(theword, cword, flags);
3280 p = cword;
3281 }
3282 else
3283 p = theword;
3284 }
3285
3286 // Add the suggestion.
3287 if (sps_flags & SPS_DOUBLE)
3288 {
3289 // Add the suggestion if the score isn't too bad.
3290 if (score <= su->su_maxscore)
3291 add_suggestion(su, &su->su_sga, p, su->su_badlen,
3292 score, 0, FALSE, slang, FALSE);
3293 }
3294 else
3295 {
3296 // Add a penalty for words in another region.
3297 if ((flags & WF_REGION)
3298 && (((unsigned)flags >> 16) & lp->lp_region) == 0)
3299 goodscore = SCORE_REGION;
3300 else
3301 goodscore = 0;
3302
3303 // Add a small penalty for changing the first letter from
3304 // lower to upper case. Helps for "tath" -> "Kath", which is
3305 // less common than "tath" -> "path". Don't do it when the
3306 // letter is the same, that has already been counted.
3307 gc = PTR2CHAR(p);
3308 if (SPELL_ISUPPER(gc))
3309 {
3310 bc = PTR2CHAR(su->su_badword);
3311 if (!SPELL_ISUPPER(bc)
3312 && SPELL_TOFOLD(bc) != SPELL_TOFOLD(gc))
3313 goodscore += SCORE_ICASE / 2;
3314 }
3315
3316 // Compute the score for the good word. This only does letter
3317 // insert/delete/swap/replace. REP items are not considered,
3318 // which may make the score a bit higher.
3319 // Use a limit for the score to make it work faster. Use
3320 // MAXSCORE(), because RESCORE() will change the score.
3321 // If the limit is very high then the iterative method is
3322 // inefficient, using an array is quicker.
3323 limit = MAXSCORE(su->su_sfmaxscore - goodscore, score);
3324 if (limit > SCORE_LIMITMAX)
3325 goodscore += spell_edit_score(slang, su->su_badword, p);
3326 else
3327 goodscore += spell_edit_score_limit(slang, su->su_badword,
3328 p, limit);
3329
3330 // When going over the limit don't bother to do the rest.
3331 if (goodscore < SCORE_MAXMAX)
3332 {
3333 // Give a bonus to words seen before.
3334 goodscore = score_wordcount_adj(slang, goodscore, p, FALSE);
3335
3336 // Add the suggestion if the score isn't too bad.
3337 goodscore = RESCORE(goodscore, score);
3338 if (goodscore <= su->su_sfmaxscore)
3339 add_suggestion(su, &su->su_ga, p, su->su_badlen,
3340 goodscore, score, TRUE, slang, TRUE);
3341 }
3342 }
3343 }
3344 // smsg("word %s (%d): %s (%d)", sftword, sftnr, theword, orgnr);
3345 }
3346}
3347
3348/*
3349 * Find word "word" in fold-case tree for "slang" and return the word number.
3350 */
3351 static int
3352soundfold_find(slang_T *slang, char_u *word)
3353{
3354 idx_T arridx = 0;
3355 int len;
3356 int wlen = 0;
3357 int c;
3358 char_u *ptr = word;
3359 char_u *byts;
3360 idx_T *idxs;
3361 int wordnr = 0;
3362
3363 byts = slang->sl_sbyts;
3364 idxs = slang->sl_sidxs;
3365
3366 for (;;)
3367 {
3368 // First byte is the number of possible bytes.
3369 len = byts[arridx++];
3370
3371 // If the first possible byte is a zero the word could end here.
3372 // If the word ends we found the word. If not skip the NUL bytes.
3373 c = ptr[wlen];
3374 if (byts[arridx] == NUL)
3375 {
3376 if (c == NUL)
3377 break;
3378
3379 // Skip over the zeros, there can be several.
3380 while (len > 0 && byts[arridx] == NUL)
3381 {
3382 ++arridx;
3383 --len;
3384 }
3385 if (len == 0)
3386 return -1; // no children, word should have ended here
3387 ++wordnr;
3388 }
3389
3390 // If the word ends we didn't find it.
3391 if (c == NUL)
3392 return -1;
3393
3394 // Perform a binary search in the list of accepted bytes.
3395 if (c == TAB) // <Tab> is handled like <Space>
3396 c = ' ';
3397 while (byts[arridx] < c)
3398 {
3399 // The word count is in the first idxs[] entry of the child.
3400 wordnr += idxs[idxs[arridx]];
3401 ++arridx;
3402 if (--len == 0) // end of the bytes, didn't find it
3403 return -1;
3404 }
3405 if (byts[arridx] != c) // didn't find the byte
3406 return -1;
3407
3408 // Continue at the child (if there is one).
3409 arridx = idxs[arridx];
3410 ++wlen;
3411
3412 // One space in the good word may stand for several spaces in the
3413 // checked word.
3414 if (c == ' ')
3415 while (ptr[wlen] == ' ' || ptr[wlen] == TAB)
3416 ++wlen;
3417 }
3418
3419 return wordnr;
3420}
3421
3422/*
3423 * Return TRUE if "c1" and "c2" are similar characters according to the MAP
3424 * lines in the .aff file.
3425 */
3426 static int
3427similar_chars(slang_T *slang, int c1, int c2)
3428{
3429 int m1, m2;
3430 char_u buf[MB_MAXBYTES + 1];
3431 hashitem_T *hi;
3432
3433 if (c1 >= 256)
3434 {
3435 buf[mb_char2bytes(c1, buf)] = 0;
3436 hi = hash_find(&slang->sl_map_hash, buf);
3437 if (HASHITEM_EMPTY(hi))
3438 m1 = 0;
3439 else
3440 m1 = mb_ptr2char(hi->hi_key + STRLEN(hi->hi_key) + 1);
3441 }
3442 else
3443 m1 = slang->sl_map_array[c1];
3444 if (m1 == 0)
3445 return FALSE;
3446
3447
3448 if (c2 >= 256)
3449 {
3450 buf[mb_char2bytes(c2, buf)] = 0;
3451 hi = hash_find(&slang->sl_map_hash, buf);
3452 if (HASHITEM_EMPTY(hi))
3453 m2 = 0;
3454 else
3455 m2 = mb_ptr2char(hi->hi_key + STRLEN(hi->hi_key) + 1);
3456 }
3457 else
3458 m2 = slang->sl_map_array[c2];
3459
3460 return m1 == m2;
3461}
3462
3463/*
3464 * Add a suggestion to the list of suggestions.
3465 * For a suggestion that is already in the list the lowest score is remembered.
3466 */
3467 static void
3468add_suggestion(
3469 suginfo_T *su,
3470 garray_T *gap, // either su_ga or su_sga
3471 char_u *goodword,
3472 int badlenarg, // len of bad word replaced with "goodword"
3473 int score,
3474 int altscore,
3475 int had_bonus, // value for st_had_bonus
3476 slang_T *slang, // language for sound folding
3477 int maxsf) // su_maxscore applies to soundfold score,
3478 // su_sfmaxscore to the total score.
3479{
3480 int goodlen; // len of goodword changed
3481 int badlen; // len of bad word changed
3482 suggest_T *stp;
3483 suggest_T new_sug;
3484 int i;
3485 char_u *pgood, *pbad;
3486
3487 // Minimize "badlen" for consistency. Avoids that changing "the the" to
3488 // "thee the" is added next to changing the first "the" the "thee".
3489 pgood = goodword + STRLEN(goodword);
3490 pbad = su->su_badptr + badlenarg;
3491 for (;;)
3492 {
3493 goodlen = (int)(pgood - goodword);
3494 badlen = (int)(pbad - su->su_badptr);
3495 if (goodlen <= 0 || badlen <= 0)
3496 break;
3497 MB_PTR_BACK(goodword, pgood);
3498 MB_PTR_BACK(su->su_badptr, pbad);
3499 if (has_mbyte)
3500 {
3501 if (mb_ptr2char(pgood) != mb_ptr2char(pbad))
3502 break;
3503 }
3504 else if (*pgood != *pbad)
3505 break;
3506 }
3507
3508 if (badlen == 0 && goodlen == 0)
3509 // goodword doesn't change anything; may happen for "the the" changing
3510 // the first "the" to itself.
3511 return;
3512
3513 if (gap->ga_len == 0)
3514 i = -1;
3515 else
3516 {
3517 // Check if the word is already there. Also check the length that is
3518 // being replaced "thes," -> "these" is a different suggestion from
3519 // "thes" -> "these".
3520 stp = &SUG(*gap, 0);
3521 for (i = gap->ga_len; --i >= 0; ++stp)
3522 if (stp->st_wordlen == goodlen
3523 && stp->st_orglen == badlen
3524 && STRNCMP(stp->st_word, goodword, goodlen) == 0)
3525 {
3526 // Found it. Remember the word with the lowest score.
3527 if (stp->st_slang == NULL)
3528 stp->st_slang = slang;
3529
3530 new_sug.st_score = score;
3531 new_sug.st_altscore = altscore;
3532 new_sug.st_had_bonus = had_bonus;
3533
3534 if (stp->st_had_bonus != had_bonus)
3535 {
3536 // Only one of the two had the soundalike score computed.
3537 // Need to do that for the other one now, otherwise the
3538 // scores can't be compared. This happens because
3539 // suggest_try_change() doesn't compute the soundalike
3540 // word to keep it fast, while some special methods set
3541 // the soundalike score to zero.
3542 if (had_bonus)
3543 rescore_one(su, stp);
3544 else
3545 {
3546 new_sug.st_word = stp->st_word;
3547 new_sug.st_wordlen = stp->st_wordlen;
3548 new_sug.st_slang = stp->st_slang;
3549 new_sug.st_orglen = badlen;
3550 rescore_one(su, &new_sug);
3551 }
3552 }
3553
3554 if (stp->st_score > new_sug.st_score)
3555 {
3556 stp->st_score = new_sug.st_score;
3557 stp->st_altscore = new_sug.st_altscore;
3558 stp->st_had_bonus = new_sug.st_had_bonus;
3559 }
3560 break;
3561 }
3562 }
3563
3564 if (i < 0 && ga_grow(gap, 1) == OK)
3565 {
3566 // Add a suggestion.
3567 stp = &SUG(*gap, gap->ga_len);
3568 stp->st_word = vim_strnsave(goodword, goodlen);
3569 if (stp->st_word != NULL)
3570 {
3571 stp->st_wordlen = goodlen;
3572 stp->st_score = score;
3573 stp->st_altscore = altscore;
3574 stp->st_had_bonus = had_bonus;
3575 stp->st_orglen = badlen;
3576 stp->st_slang = slang;
3577 ++gap->ga_len;
3578
3579 // If we have too many suggestions now, sort the list and keep
3580 // the best suggestions.
3581 if (gap->ga_len > SUG_MAX_COUNT(su))
3582 {
3583 if (maxsf)
3584 su->su_sfmaxscore = cleanup_suggestions(gap,
3585 su->su_sfmaxscore, SUG_CLEAN_COUNT(su));
3586 else
3587 su->su_maxscore = cleanup_suggestions(gap,
3588 su->su_maxscore, SUG_CLEAN_COUNT(su));
3589 }
3590 }
3591 }
3592}
3593
3594/*
3595 * Suggestions may in fact be flagged as errors. Esp. for banned words and
3596 * for split words, such as "the the". Remove these from the list here.
3597 */
3598 static void
3599check_suggestions(
3600 suginfo_T *su,
3601 garray_T *gap) // either su_ga or su_sga
3602{
3603 suggest_T *stp;
3604 int i;
3605 char_u longword[MAXWLEN + 1];
3606 int len;
3607 hlf_T attr;
3608
3609 stp = &SUG(*gap, 0);
3610 for (i = gap->ga_len - 1; i >= 0; --i)
3611 {
3612 // Need to append what follows to check for "the the".
3613 vim_strncpy(longword, stp[i].st_word, MAXWLEN);
3614 len = stp[i].st_wordlen;
3615 vim_strncpy(longword + len, su->su_badptr + stp[i].st_orglen,
3616 MAXWLEN - len);
3617 attr = HLF_COUNT;
3618 (void)spell_check(curwin, longword, &attr, NULL, FALSE);
3619 if (attr != HLF_COUNT)
3620 {
3621 // Remove this entry.
3622 vim_free(stp[i].st_word);
3623 --gap->ga_len;
3624 if (i < gap->ga_len)
3625 mch_memmove(stp + i, stp + i + 1,
3626 sizeof(suggest_T) * (gap->ga_len - i));
3627 }
3628 }
3629}
3630
3631
3632/*
3633 * Add a word to be banned.
3634 */
3635 static void
3636add_banned(
3637 suginfo_T *su,
3638 char_u *word)
3639{
3640 char_u *s;
3641 hash_T hash;
3642 hashitem_T *hi;
3643
3644 hash = hash_hash(word);
3645 hi = hash_lookup(&su->su_banned, word, hash);
3646 if (HASHITEM_EMPTY(hi))
3647 {
3648 s = vim_strsave(word);
3649 if (s != NULL)
3650 hash_add_item(&su->su_banned, hi, s, hash);
3651 }
3652}
3653
3654/*
3655 * Recompute the score for all suggestions if sound-folding is possible. This
3656 * is slow, thus only done for the final results.
3657 */
3658 static void
3659rescore_suggestions(suginfo_T *su)
3660{
3661 int i;
3662
3663 if (su->su_sallang != NULL)
3664 for (i = 0; i < su->su_ga.ga_len; ++i)
3665 rescore_one(su, &SUG(su->su_ga, i));
3666}
3667
3668/*
3669 * Recompute the score for one suggestion if sound-folding is possible.
3670 */
3671 static void
3672rescore_one(suginfo_T *su, suggest_T *stp)
3673{
3674 slang_T *slang = stp->st_slang;
3675 char_u sal_badword[MAXWLEN];
3676 char_u *p;
3677
3678 // Only rescore suggestions that have no sal score yet and do have a
3679 // language.
3680 if (slang != NULL && slang->sl_sal.ga_len > 0 && !stp->st_had_bonus)
3681 {
3682 if (slang == su->su_sallang)
3683 p = su->su_sal_badword;
3684 else
3685 {
3686 spell_soundfold(slang, su->su_fbadword, TRUE, sal_badword);
3687 p = sal_badword;
3688 }
3689
3690 stp->st_altscore = stp_sal_score(stp, su, slang, p);
3691 if (stp->st_altscore == SCORE_MAXMAX)
3692 stp->st_altscore = SCORE_BIG;
3693 stp->st_score = RESCORE(stp->st_score, stp->st_altscore);
3694 stp->st_had_bonus = TRUE;
3695 }
3696}
3697
3698static int sug_compare(const void *s1, const void *s2);
3699
3700/*
3701 * Function given to qsort() to sort the suggestions on st_score.
3702 * First on "st_score", then "st_altscore" then alphabetically.
3703 */
3704 static int
3705sug_compare(const void *s1, const void *s2)
3706{
3707 suggest_T *p1 = (suggest_T *)s1;
3708 suggest_T *p2 = (suggest_T *)s2;
3709 int n = p1->st_score - p2->st_score;
3710
3711 if (n == 0)
3712 {
3713 n = p1->st_altscore - p2->st_altscore;
3714 if (n == 0)
3715 n = STRICMP(p1->st_word, p2->st_word);
3716 }
3717 return n;
3718}
3719
3720/*
3721 * Cleanup the suggestions:
3722 * - Sort on score.
3723 * - Remove words that won't be displayed.
3724 * Returns the maximum score in the list or "maxscore" unmodified.
3725 */
3726 static int
3727cleanup_suggestions(
3728 garray_T *gap,
3729 int maxscore,
3730 int keep) // nr of suggestions to keep
3731{
3732 suggest_T *stp = &SUG(*gap, 0);
3733 int i;
3734
Bram Moolenaarbb65a562020-03-15 18:15:03 +01003735 if (gap->ga_len > 0)
Bram Moolenaar46a426c2019-09-27 12:41:56 +02003736 {
Bram Moolenaarbb65a562020-03-15 18:15:03 +01003737 // Sort the list.
3738 qsort(gap->ga_data, (size_t)gap->ga_len, sizeof(suggest_T),
3739 sug_compare);
3740
3741 // Truncate the list to the number of suggestions that will be
3742 // displayed.
3743 if (gap->ga_len > keep)
3744 {
3745 for (i = keep; i < gap->ga_len; ++i)
3746 vim_free(stp[i].st_word);
3747 gap->ga_len = keep;
3748 if (keep >= 1)
3749 return stp[keep - 1].st_score;
3750 }
Bram Moolenaar46a426c2019-09-27 12:41:56 +02003751 }
3752 return maxscore;
3753}
3754
3755/*
3756 * Compute a score for two sound-a-like words.
3757 * This permits up to two inserts/deletes/swaps/etc. to keep things fast.
3758 * Instead of a generic loop we write out the code. That keeps it fast by
3759 * avoiding checks that will not be possible.
3760 */
3761 static int
3762soundalike_score(
3763 char_u *goodstart, // sound-folded good word
3764 char_u *badstart) // sound-folded bad word
3765{
3766 char_u *goodsound = goodstart;
3767 char_u *badsound = badstart;
3768 int goodlen;
3769 int badlen;
3770 int n;
3771 char_u *pl, *ps;
3772 char_u *pl2, *ps2;
3773 int score = 0;
3774
3775 // Adding/inserting "*" at the start (word starts with vowel) shouldn't be
3776 // counted so much, vowels halfway the word aren't counted at all.
3777 if ((*badsound == '*' || *goodsound == '*') && *badsound != *goodsound)
3778 {
3779 if ((badsound[0] == NUL && goodsound[1] == NUL)
3780 || (goodsound[0] == NUL && badsound[1] == NUL))
3781 // changing word with vowel to word without a sound
3782 return SCORE_DEL;
3783 if (badsound[0] == NUL || goodsound[0] == NUL)
3784 // more than two changes
3785 return SCORE_MAXMAX;
3786
3787 if (badsound[1] == goodsound[1]
3788 || (badsound[1] != NUL
3789 && goodsound[1] != NUL
3790 && badsound[2] == goodsound[2]))
3791 {
3792 // handle like a substitute
3793 }
3794 else
3795 {
3796 score = 2 * SCORE_DEL / 3;
3797 if (*badsound == '*')
3798 ++badsound;
3799 else
3800 ++goodsound;
3801 }
3802 }
3803
3804 goodlen = (int)STRLEN(goodsound);
3805 badlen = (int)STRLEN(badsound);
3806
3807 // Return quickly if the lengths are too different to be fixed by two
3808 // changes.
3809 n = goodlen - badlen;
3810 if (n < -2 || n > 2)
3811 return SCORE_MAXMAX;
3812
3813 if (n > 0)
3814 {
3815 pl = goodsound; // goodsound is longest
3816 ps = badsound;
3817 }
3818 else
3819 {
3820 pl = badsound; // badsound is longest
3821 ps = goodsound;
3822 }
3823
3824 // Skip over the identical part.
3825 while (*pl == *ps && *pl != NUL)
3826 {
3827 ++pl;
3828 ++ps;
3829 }
3830
3831 switch (n)
3832 {
3833 case -2:
3834 case 2:
3835 // Must delete two characters from "pl".
3836 ++pl; // first delete
3837 while (*pl == *ps)
3838 {
3839 ++pl;
3840 ++ps;
3841 }
3842 // strings must be equal after second delete
3843 if (STRCMP(pl + 1, ps) == 0)
3844 return score + SCORE_DEL * 2;
3845
3846 // Failed to compare.
3847 break;
3848
3849 case -1:
3850 case 1:
3851 // Minimal one delete from "pl" required.
3852
3853 // 1: delete
3854 pl2 = pl + 1;
3855 ps2 = ps;
3856 while (*pl2 == *ps2)
3857 {
3858 if (*pl2 == NUL) // reached the end
3859 return score + SCORE_DEL;
3860 ++pl2;
3861 ++ps2;
3862 }
3863
3864 // 2: delete then swap, then rest must be equal
3865 if (pl2[0] == ps2[1] && pl2[1] == ps2[0]
3866 && STRCMP(pl2 + 2, ps2 + 2) == 0)
3867 return score + SCORE_DEL + SCORE_SWAP;
3868
3869 // 3: delete then substitute, then the rest must be equal
3870 if (STRCMP(pl2 + 1, ps2 + 1) == 0)
3871 return score + SCORE_DEL + SCORE_SUBST;
3872
3873 // 4: first swap then delete
3874 if (pl[0] == ps[1] && pl[1] == ps[0])
3875 {
3876 pl2 = pl + 2; // swap, skip two chars
3877 ps2 = ps + 2;
3878 while (*pl2 == *ps2)
3879 {
3880 ++pl2;
3881 ++ps2;
3882 }
3883 // delete a char and then strings must be equal
3884 if (STRCMP(pl2 + 1, ps2) == 0)
3885 return score + SCORE_SWAP + SCORE_DEL;
3886 }
3887
3888 // 5: first substitute then delete
3889 pl2 = pl + 1; // substitute, skip one char
3890 ps2 = ps + 1;
3891 while (*pl2 == *ps2)
3892 {
3893 ++pl2;
3894 ++ps2;
3895 }
3896 // delete a char and then strings must be equal
3897 if (STRCMP(pl2 + 1, ps2) == 0)
3898 return score + SCORE_SUBST + SCORE_DEL;
3899
3900 // Failed to compare.
3901 break;
3902
3903 case 0:
3904 // Lengths are equal, thus changes must result in same length: An
3905 // insert is only possible in combination with a delete.
3906 // 1: check if for identical strings
3907 if (*pl == NUL)
3908 return score;
3909
3910 // 2: swap
3911 if (pl[0] == ps[1] && pl[1] == ps[0])
3912 {
3913 pl2 = pl + 2; // swap, skip two chars
3914 ps2 = ps + 2;
3915 while (*pl2 == *ps2)
3916 {
3917 if (*pl2 == NUL) // reached the end
3918 return score + SCORE_SWAP;
3919 ++pl2;
3920 ++ps2;
3921 }
3922 // 3: swap and swap again
3923 if (pl2[0] == ps2[1] && pl2[1] == ps2[0]
3924 && STRCMP(pl2 + 2, ps2 + 2) == 0)
3925 return score + SCORE_SWAP + SCORE_SWAP;
3926
3927 // 4: swap and substitute
3928 if (STRCMP(pl2 + 1, ps2 + 1) == 0)
3929 return score + SCORE_SWAP + SCORE_SUBST;
3930 }
3931
3932 // 5: substitute
3933 pl2 = pl + 1;
3934 ps2 = ps + 1;
3935 while (*pl2 == *ps2)
3936 {
3937 if (*pl2 == NUL) // reached the end
3938 return score + SCORE_SUBST;
3939 ++pl2;
3940 ++ps2;
3941 }
3942
3943 // 6: substitute and swap
3944 if (pl2[0] == ps2[1] && pl2[1] == ps2[0]
3945 && STRCMP(pl2 + 2, ps2 + 2) == 0)
3946 return score + SCORE_SUBST + SCORE_SWAP;
3947
3948 // 7: substitute and substitute
3949 if (STRCMP(pl2 + 1, ps2 + 1) == 0)
3950 return score + SCORE_SUBST + SCORE_SUBST;
3951
3952 // 8: insert then delete
3953 pl2 = pl;
3954 ps2 = ps + 1;
3955 while (*pl2 == *ps2)
3956 {
3957 ++pl2;
3958 ++ps2;
3959 }
3960 if (STRCMP(pl2 + 1, ps2) == 0)
3961 return score + SCORE_INS + SCORE_DEL;
3962
3963 // 9: delete then insert
3964 pl2 = pl + 1;
3965 ps2 = ps;
3966 while (*pl2 == *ps2)
3967 {
3968 ++pl2;
3969 ++ps2;
3970 }
3971 if (STRCMP(pl2, ps2 + 1) == 0)
3972 return score + SCORE_INS + SCORE_DEL;
3973
3974 // Failed to compare.
3975 break;
3976 }
3977
3978 return SCORE_MAXMAX;
3979}
3980
3981/*
3982 * Compute the "edit distance" to turn "badword" into "goodword". The less
3983 * deletes/inserts/substitutes/swaps are required the lower the score.
3984 *
3985 * The algorithm is described by Du and Chang, 1992.
3986 * The implementation of the algorithm comes from Aspell editdist.cpp,
3987 * edit_distance(). It has been converted from C++ to C and modified to
3988 * support multi-byte characters.
3989 */
3990 static int
3991spell_edit_score(
3992 slang_T *slang,
3993 char_u *badword,
3994 char_u *goodword)
3995{
3996 int *cnt;
3997 int badlen, goodlen; // lengths including NUL
3998 int j, i;
3999 int t;
4000 int bc, gc;
4001 int pbc, pgc;
4002 char_u *p;
4003 int wbadword[MAXWLEN];
4004 int wgoodword[MAXWLEN];
4005
4006 if (has_mbyte)
4007 {
4008 // Get the characters from the multi-byte strings and put them in an
4009 // int array for easy access.
4010 for (p = badword, badlen = 0; *p != NUL; )
4011 wbadword[badlen++] = mb_cptr2char_adv(&p);
4012 wbadword[badlen++] = 0;
4013 for (p = goodword, goodlen = 0; *p != NUL; )
4014 wgoodword[goodlen++] = mb_cptr2char_adv(&p);
4015 wgoodword[goodlen++] = 0;
4016 }
4017 else
4018 {
4019 badlen = (int)STRLEN(badword) + 1;
4020 goodlen = (int)STRLEN(goodword) + 1;
4021 }
4022
4023 // We use "cnt" as an array: CNT(badword_idx, goodword_idx).
4024#define CNT(a, b) cnt[(a) + (b) * (badlen + 1)]
4025 cnt = ALLOC_MULT(int, (badlen + 1) * (goodlen + 1));
4026 if (cnt == NULL)
4027 return 0; // out of memory
4028
4029 CNT(0, 0) = 0;
4030 for (j = 1; j <= goodlen; ++j)
4031 CNT(0, j) = CNT(0, j - 1) + SCORE_INS;
4032
4033 for (i = 1; i <= badlen; ++i)
4034 {
4035 CNT(i, 0) = CNT(i - 1, 0) + SCORE_DEL;
4036 for (j = 1; j <= goodlen; ++j)
4037 {
4038 if (has_mbyte)
4039 {
4040 bc = wbadword[i - 1];
4041 gc = wgoodword[j - 1];
4042 }
4043 else
4044 {
4045 bc = badword[i - 1];
4046 gc = goodword[j - 1];
4047 }
4048 if (bc == gc)
4049 CNT(i, j) = CNT(i - 1, j - 1);
4050 else
4051 {
4052 // Use a better score when there is only a case difference.
4053 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc))
4054 CNT(i, j) = SCORE_ICASE + CNT(i - 1, j - 1);
4055 else
4056 {
4057 // For a similar character use SCORE_SIMILAR.
4058 if (slang != NULL
4059 && slang->sl_has_map
4060 && similar_chars(slang, gc, bc))
4061 CNT(i, j) = SCORE_SIMILAR + CNT(i - 1, j - 1);
4062 else
4063 CNT(i, j) = SCORE_SUBST + CNT(i - 1, j - 1);
4064 }
4065
4066 if (i > 1 && j > 1)
4067 {
4068 if (has_mbyte)
4069 {
4070 pbc = wbadword[i - 2];
4071 pgc = wgoodword[j - 2];
4072 }
4073 else
4074 {
4075 pbc = badword[i - 2];
4076 pgc = goodword[j - 2];
4077 }
4078 if (bc == pgc && pbc == gc)
4079 {
4080 t = SCORE_SWAP + CNT(i - 2, j - 2);
4081 if (t < CNT(i, j))
4082 CNT(i, j) = t;
4083 }
4084 }
4085 t = SCORE_DEL + CNT(i - 1, j);
4086 if (t < CNT(i, j))
4087 CNT(i, j) = t;
4088 t = SCORE_INS + CNT(i, j - 1);
4089 if (t < CNT(i, j))
4090 CNT(i, j) = t;
4091 }
4092 }
4093 }
4094
4095 i = CNT(badlen - 1, goodlen - 1);
4096 vim_free(cnt);
4097 return i;
4098}
4099
4100typedef struct
4101{
4102 int badi;
4103 int goodi;
4104 int score;
4105} limitscore_T;
4106
4107/*
4108 * Like spell_edit_score(), but with a limit on the score to make it faster.
4109 * May return SCORE_MAXMAX when the score is higher than "limit".
4110 *
4111 * This uses a stack for the edits still to be tried.
4112 * The idea comes from Aspell leditdist.cpp. Rewritten in C and added support
4113 * for multi-byte characters.
4114 */
4115 static int
4116spell_edit_score_limit(
4117 slang_T *slang,
4118 char_u *badword,
4119 char_u *goodword,
4120 int limit)
4121{
4122 limitscore_T stack[10]; // allow for over 3 * 2 edits
4123 int stackidx;
4124 int bi, gi;
4125 int bi2, gi2;
4126 int bc, gc;
4127 int score;
4128 int score_off;
4129 int minscore;
4130 int round;
4131
4132 // Multi-byte characters require a bit more work, use a different function
4133 // to avoid testing "has_mbyte" quite often.
4134 if (has_mbyte)
4135 return spell_edit_score_limit_w(slang, badword, goodword, limit);
4136
4137 // The idea is to go from start to end over the words. So long as
4138 // characters are equal just continue, this always gives the lowest score.
4139 // When there is a difference try several alternatives. Each alternative
4140 // increases "score" for the edit distance. Some of the alternatives are
4141 // pushed unto a stack and tried later, some are tried right away. At the
4142 // end of the word the score for one alternative is known. The lowest
4143 // possible score is stored in "minscore".
4144 stackidx = 0;
4145 bi = 0;
4146 gi = 0;
4147 score = 0;
4148 minscore = limit + 1;
4149
4150 for (;;)
4151 {
4152 // Skip over an equal part, score remains the same.
4153 for (;;)
4154 {
4155 bc = badword[bi];
4156 gc = goodword[gi];
4157 if (bc != gc) // stop at a char that's different
4158 break;
4159 if (bc == NUL) // both words end
4160 {
4161 if (score < minscore)
4162 minscore = score;
4163 goto pop; // do next alternative
4164 }
4165 ++bi;
4166 ++gi;
4167 }
4168
4169 if (gc == NUL) // goodword ends, delete badword chars
4170 {
4171 do
4172 {
4173 if ((score += SCORE_DEL) >= minscore)
4174 goto pop; // do next alternative
4175 } while (badword[++bi] != NUL);
4176 minscore = score;
4177 }
4178 else if (bc == NUL) // badword ends, insert badword chars
4179 {
4180 do
4181 {
4182 if ((score += SCORE_INS) >= minscore)
4183 goto pop; // do next alternative
4184 } while (goodword[++gi] != NUL);
4185 minscore = score;
4186 }
4187 else // both words continue
4188 {
4189 // If not close to the limit, perform a change. Only try changes
4190 // that may lead to a lower score than "minscore".
4191 // round 0: try deleting a char from badword
4192 // round 1: try inserting a char in badword
4193 for (round = 0; round <= 1; ++round)
4194 {
4195 score_off = score + (round == 0 ? SCORE_DEL : SCORE_INS);
4196 if (score_off < minscore)
4197 {
4198 if (score_off + SCORE_EDIT_MIN >= minscore)
4199 {
4200 // Near the limit, rest of the words must match. We
4201 // can check that right now, no need to push an item
4202 // onto the stack.
4203 bi2 = bi + 1 - round;
4204 gi2 = gi + round;
4205 while (goodword[gi2] == badword[bi2])
4206 {
4207 if (goodword[gi2] == NUL)
4208 {
4209 minscore = score_off;
4210 break;
4211 }
4212 ++bi2;
4213 ++gi2;
4214 }
4215 }
4216 else
4217 {
4218 // try deleting/inserting a character later
4219 stack[stackidx].badi = bi + 1 - round;
4220 stack[stackidx].goodi = gi + round;
4221 stack[stackidx].score = score_off;
4222 ++stackidx;
4223 }
4224 }
4225 }
4226
4227 if (score + SCORE_SWAP < minscore)
4228 {
4229 // If swapping two characters makes a match then the
4230 // substitution is more expensive, thus there is no need to
4231 // try both.
4232 if (gc == badword[bi + 1] && bc == goodword[gi + 1])
4233 {
4234 // Swap two characters, that is: skip them.
4235 gi += 2;
4236 bi += 2;
4237 score += SCORE_SWAP;
4238 continue;
4239 }
4240 }
4241
4242 // Substitute one character for another which is the same
4243 // thing as deleting a character from both goodword and badword.
4244 // Use a better score when there is only a case difference.
4245 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc))
4246 score += SCORE_ICASE;
4247 else
4248 {
4249 // For a similar character use SCORE_SIMILAR.
4250 if (slang != NULL
4251 && slang->sl_has_map
4252 && similar_chars(slang, gc, bc))
4253 score += SCORE_SIMILAR;
4254 else
4255 score += SCORE_SUBST;
4256 }
4257
4258 if (score < minscore)
4259 {
4260 // Do the substitution.
4261 ++gi;
4262 ++bi;
4263 continue;
4264 }
4265 }
4266pop:
4267 // Get here to try the next alternative, pop it from the stack.
4268 if (stackidx == 0) // stack is empty, finished
4269 break;
4270
4271 // pop an item from the stack
4272 --stackidx;
4273 gi = stack[stackidx].goodi;
4274 bi = stack[stackidx].badi;
4275 score = stack[stackidx].score;
4276 }
4277
4278 // When the score goes over "limit" it may actually be much higher.
4279 // Return a very large number to avoid going below the limit when giving a
4280 // bonus.
4281 if (minscore > limit)
4282 return SCORE_MAXMAX;
4283 return minscore;
4284}
4285
4286/*
4287 * Multi-byte version of spell_edit_score_limit().
4288 * Keep it in sync with the above!
4289 */
4290 static int
4291spell_edit_score_limit_w(
4292 slang_T *slang,
4293 char_u *badword,
4294 char_u *goodword,
4295 int limit)
4296{
4297 limitscore_T stack[10]; // allow for over 3 * 2 edits
4298 int stackidx;
4299 int bi, gi;
4300 int bi2, gi2;
4301 int bc, gc;
4302 int score;
4303 int score_off;
4304 int minscore;
4305 int round;
4306 char_u *p;
4307 int wbadword[MAXWLEN];
4308 int wgoodword[MAXWLEN];
4309
4310 // Get the characters from the multi-byte strings and put them in an
4311 // int array for easy access.
4312 bi = 0;
4313 for (p = badword; *p != NUL; )
4314 wbadword[bi++] = mb_cptr2char_adv(&p);
4315 wbadword[bi++] = 0;
4316 gi = 0;
4317 for (p = goodword; *p != NUL; )
4318 wgoodword[gi++] = mb_cptr2char_adv(&p);
4319 wgoodword[gi++] = 0;
4320
4321 // The idea is to go from start to end over the words. So long as
4322 // characters are equal just continue, this always gives the lowest score.
4323 // When there is a difference try several alternatives. Each alternative
4324 // increases "score" for the edit distance. Some of the alternatives are
4325 // pushed unto a stack and tried later, some are tried right away. At the
4326 // end of the word the score for one alternative is known. The lowest
4327 // possible score is stored in "minscore".
4328 stackidx = 0;
4329 bi = 0;
4330 gi = 0;
4331 score = 0;
4332 minscore = limit + 1;
4333
4334 for (;;)
4335 {
4336 // Skip over an equal part, score remains the same.
4337 for (;;)
4338 {
4339 bc = wbadword[bi];
4340 gc = wgoodword[gi];
4341
4342 if (bc != gc) // stop at a char that's different
4343 break;
4344 if (bc == NUL) // both words end
4345 {
4346 if (score < minscore)
4347 minscore = score;
4348 goto pop; // do next alternative
4349 }
4350 ++bi;
4351 ++gi;
4352 }
4353
4354 if (gc == NUL) // goodword ends, delete badword chars
4355 {
4356 do
4357 {
4358 if ((score += SCORE_DEL) >= minscore)
4359 goto pop; // do next alternative
4360 } while (wbadword[++bi] != NUL);
4361 minscore = score;
4362 }
4363 else if (bc == NUL) // badword ends, insert badword chars
4364 {
4365 do
4366 {
4367 if ((score += SCORE_INS) >= minscore)
4368 goto pop; // do next alternative
4369 } while (wgoodword[++gi] != NUL);
4370 minscore = score;
4371 }
4372 else // both words continue
4373 {
4374 // If not close to the limit, perform a change. Only try changes
4375 // that may lead to a lower score than "minscore".
4376 // round 0: try deleting a char from badword
4377 // round 1: try inserting a char in badword
4378 for (round = 0; round <= 1; ++round)
4379 {
4380 score_off = score + (round == 0 ? SCORE_DEL : SCORE_INS);
4381 if (score_off < minscore)
4382 {
4383 if (score_off + SCORE_EDIT_MIN >= minscore)
4384 {
4385 // Near the limit, rest of the words must match. We
4386 // can check that right now, no need to push an item
4387 // onto the stack.
4388 bi2 = bi + 1 - round;
4389 gi2 = gi + round;
4390 while (wgoodword[gi2] == wbadword[bi2])
4391 {
4392 if (wgoodword[gi2] == NUL)
4393 {
4394 minscore = score_off;
4395 break;
4396 }
4397 ++bi2;
4398 ++gi2;
4399 }
4400 }
4401 else
4402 {
4403 // try deleting a character from badword later
4404 stack[stackidx].badi = bi + 1 - round;
4405 stack[stackidx].goodi = gi + round;
4406 stack[stackidx].score = score_off;
4407 ++stackidx;
4408 }
4409 }
4410 }
4411
4412 if (score + SCORE_SWAP < minscore)
4413 {
4414 // If swapping two characters makes a match then the
4415 // substitution is more expensive, thus there is no need to
4416 // try both.
4417 if (gc == wbadword[bi + 1] && bc == wgoodword[gi + 1])
4418 {
4419 // Swap two characters, that is: skip them.
4420 gi += 2;
4421 bi += 2;
4422 score += SCORE_SWAP;
4423 continue;
4424 }
4425 }
4426
4427 // Substitute one character for another which is the same
4428 // thing as deleting a character from both goodword and badword.
4429 // Use a better score when there is only a case difference.
4430 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc))
4431 score += SCORE_ICASE;
4432 else
4433 {
4434 // For a similar character use SCORE_SIMILAR.
4435 if (slang != NULL
4436 && slang->sl_has_map
4437 && similar_chars(slang, gc, bc))
4438 score += SCORE_SIMILAR;
4439 else
4440 score += SCORE_SUBST;
4441 }
4442
4443 if (score < minscore)
4444 {
4445 // Do the substitution.
4446 ++gi;
4447 ++bi;
4448 continue;
4449 }
4450 }
4451pop:
4452 // Get here to try the next alternative, pop it from the stack.
4453 if (stackidx == 0) // stack is empty, finished
4454 break;
4455
4456 // pop an item from the stack
4457 --stackidx;
4458 gi = stack[stackidx].goodi;
4459 bi = stack[stackidx].badi;
4460 score = stack[stackidx].score;
4461 }
4462
4463 // When the score goes over "limit" it may actually be much higher.
4464 // Return a very large number to avoid going below the limit when giving a
4465 // bonus.
4466 if (minscore > limit)
4467 return SCORE_MAXMAX;
4468 return minscore;
4469}
4470#endif // FEAT_SPELL