blob: 2951eea2eaa2641585d66ca46c7b63fca1f42031 [file] [log] [blame]
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001/* vi:set ts=8 sts=4 sw=4 noet:
2 *
3 * VIM - Vi IMproved by Bram Moolenaar
4 *
5 * Do ":help uganda" in Vim to read copying and usage conditions.
6 * Do ":help credits" in Vim to see a list of people who contributed.
7 * See README.txt for an overview of the Vim source code.
8 */
9
10/*
11 * spellsuggest.c: functions for spelling suggestions
12 */
13
14#include "vim.h"
15
16#if defined(FEAT_SPELL) || defined(PROTO)
17
18/*
19 * Use this to adjust the score after finding suggestions, based on the
20 * suggested word sounding like the bad word. This is much faster than doing
21 * it for every possible suggestion.
22 * Disadvantage: When "the" is typed as "hte" it sounds quite different ("@"
23 * vs "ht") and goes down in the list.
24 * Used when 'spellsuggest' is set to "best".
25 */
26#define RESCORE(word_score, sound_score) ((3 * word_score + sound_score) / 4)
27
28/*
29 * Do the opposite: based on a maximum end score and a known sound score,
30 * compute the maximum word score that can be used.
31 */
32#define MAXSCORE(word_score, sound_score) ((4 * word_score - sound_score) / 3)
33
34// only used for su_badflags
35#define WF_MIXCAP 0x20 // mix of upper and lower case: macaRONI
36
37/*
38 * Information used when looking for suggestions.
39 */
40typedef struct suginfo_S
41{
42 garray_T su_ga; // suggestions, contains "suggest_T"
43 int su_maxcount; // max. number of suggestions displayed
44 int su_maxscore; // maximum score for adding to su_ga
45 int su_sfmaxscore; // idem, for when doing soundfold words
46 garray_T su_sga; // like su_ga, sound-folded scoring
47 char_u *su_badptr; // start of bad word in line
48 int su_badlen; // length of detected bad word in line
49 int su_badflags; // caps flags for bad word
50 char_u su_badword[MAXWLEN]; // bad word truncated at su_badlen
51 char_u su_fbadword[MAXWLEN]; // su_badword case-folded
52 char_u su_sal_badword[MAXWLEN]; // su_badword soundfolded
53 hashtab_T su_banned; // table with banned words
54 slang_T *su_sallang; // default language for sound folding
55} suginfo_T;
56
57// One word suggestion. Used in "si_ga".
58typedef struct suggest_S
59{
60 char_u *st_word; // suggested word, allocated string
61 int st_wordlen; // STRLEN(st_word)
62 int st_orglen; // length of replaced text
63 int st_score; // lower is better
64 int st_altscore; // used when st_score compares equal
65 int st_salscore; // st_score is for soundalike
66 int st_had_bonus; // bonus already included in score
67 slang_T *st_slang; // language used for sound folding
68} suggest_T;
69
70#define SUG(ga, i) (((suggest_T *)(ga).ga_data)[i])
71
72// TRUE if a word appears in the list of banned words.
73#define WAS_BANNED(su, word) (!HASHITEM_EMPTY(hash_find(&su->su_banned, word)))
74
75// Number of suggestions kept when cleaning up. We need to keep more than
76// what is displayed, because when rescore_suggestions() is called the score
77// may change and wrong suggestions may be removed later.
78#define SUG_CLEAN_COUNT(su) ((su)->su_maxcount < 130 ? 150 : (su)->su_maxcount + 20)
79
80// Threshold for sorting and cleaning up suggestions. Don't want to keep lots
81// of suggestions that are not going to be displayed.
82#define SUG_MAX_COUNT(su) (SUG_CLEAN_COUNT(su) + 50)
83
84// score for various changes
85#define SCORE_SPLIT 149 // split bad word
86#define SCORE_SPLIT_NO 249 // split bad word with NOSPLITSUGS
87#define SCORE_ICASE 52 // slightly different case
88#define SCORE_REGION 200 // word is for different region
89#define SCORE_RARE 180 // rare word
90#define SCORE_SWAP 75 // swap two characters
91#define SCORE_SWAP3 110 // swap two characters in three
92#define SCORE_REP 65 // REP replacement
93#define SCORE_SUBST 93 // substitute a character
94#define SCORE_SIMILAR 33 // substitute a similar character
95#define SCORE_SUBCOMP 33 // substitute a composing character
96#define SCORE_DEL 94 // delete a character
97#define SCORE_DELDUP 66 // delete a duplicated character
98#define SCORE_DELCOMP 28 // delete a composing character
99#define SCORE_INS 96 // insert a character
100#define SCORE_INSDUP 67 // insert a duplicate character
101#define SCORE_INSCOMP 30 // insert a composing character
102#define SCORE_NONWORD 103 // change non-word to word char
103
104#define SCORE_FILE 30 // suggestion from a file
105#define SCORE_MAXINIT 350 // Initial maximum score: higher == slower.
106 // 350 allows for about three changes.
107
108#define SCORE_COMMON1 30 // subtracted for words seen before
109#define SCORE_COMMON2 40 // subtracted for words often seen
110#define SCORE_COMMON3 50 // subtracted for words very often seen
111#define SCORE_THRES2 10 // word count threshold for COMMON2
112#define SCORE_THRES3 100 // word count threshold for COMMON3
113
114// When trying changed soundfold words it becomes slow when trying more than
115// two changes. With less then two changes it's slightly faster but we miss a
116// few good suggestions. In rare cases we need to try three of four changes.
117#define SCORE_SFMAX1 200 // maximum score for first try
118#define SCORE_SFMAX2 300 // maximum score for second try
119#define SCORE_SFMAX3 400 // maximum score for third try
120
121#define SCORE_BIG SCORE_INS * 3 // big difference
122#define SCORE_MAXMAX 999999 // accept any score
123#define SCORE_LIMITMAX 350 // for spell_edit_score_limit()
124
125// for spell_edit_score_limit() we need to know the minimum value of
126// SCORE_ICASE, SCORE_SWAP, SCORE_DEL, SCORE_SIMILAR and SCORE_INS
127#define SCORE_EDIT_MIN SCORE_SIMILAR
128
129/*
130 * For finding suggestions: At each node in the tree these states are tried:
131 */
132typedef enum
133{
134 STATE_START = 0, // At start of node check for NUL bytes (goodword
135 // ends); if badword ends there is a match, otherwise
136 // try splitting word.
137 STATE_NOPREFIX, // try without prefix
138 STATE_SPLITUNDO, // Undo splitting.
139 STATE_ENDNUL, // Past NUL bytes at start of the node.
140 STATE_PLAIN, // Use each byte of the node.
141 STATE_DEL, // Delete a byte from the bad word.
142 STATE_INS_PREP, // Prepare for inserting bytes.
143 STATE_INS, // Insert a byte in the bad word.
144 STATE_SWAP, // Swap two bytes.
145 STATE_UNSWAP, // Undo swap two characters.
146 STATE_SWAP3, // Swap two characters over three.
147 STATE_UNSWAP3, // Undo Swap two characters over three.
148 STATE_UNROT3L, // Undo rotate three characters left
149 STATE_UNROT3R, // Undo rotate three characters right
150 STATE_REP_INI, // Prepare for using REP items.
151 STATE_REP, // Use matching REP items from the .aff file.
152 STATE_REP_UNDO, // Undo a REP item replacement.
153 STATE_FINAL // End of this node.
154} state_T;
155
156/*
157 * Struct to keep the state at each level in suggest_try_change().
158 */
159typedef struct trystate_S
160{
161 state_T ts_state; // state at this level, STATE_
162 int ts_score; // score
163 idx_T ts_arridx; // index in tree array, start of node
164 short ts_curi; // index in list of child nodes
165 char_u ts_fidx; // index in fword[], case-folded bad word
166 char_u ts_fidxtry; // ts_fidx at which bytes may be changed
167 char_u ts_twordlen; // valid length of tword[]
168 char_u ts_prefixdepth; // stack depth for end of prefix or
169 // PFD_PREFIXTREE or PFD_NOPREFIX
170 char_u ts_flags; // TSF_ flags
171 char_u ts_tcharlen; // number of bytes in tword character
172 char_u ts_tcharidx; // current byte index in tword character
173 char_u ts_isdiff; // DIFF_ values
174 char_u ts_fcharstart; // index in fword where badword char started
175 char_u ts_prewordlen; // length of word in "preword[]"
176 char_u ts_splitoff; // index in "tword" after last split
177 char_u ts_splitfidx; // "ts_fidx" at word split
178 char_u ts_complen; // nr of compound words used
179 char_u ts_compsplit; // index for "compflags" where word was spit
180 char_u ts_save_badflags; // su_badflags saved here
181 char_u ts_delidx; // index in fword for char that was deleted,
182 // valid when "ts_flags" has TSF_DIDDEL
183} trystate_T;
184
185// values for ts_isdiff
186#define DIFF_NONE 0 // no different byte (yet)
187#define DIFF_YES 1 // different byte found
188#define DIFF_INSERT 2 // inserting character
189
190// values for ts_flags
191#define TSF_PREFIXOK 1 // already checked that prefix is OK
192#define TSF_DIDSPLIT 2 // tried split at this point
193#define TSF_DIDDEL 4 // did a delete, "ts_delidx" has index
194
195// special values ts_prefixdepth
196#define PFD_NOPREFIX 0xff // not using prefixes
197#define PFD_PREFIXTREE 0xfe // walking through the prefix tree
198#define PFD_NOTSPECIAL 0xfd // highest value that's not special
199
200static void spell_find_suggest(char_u *badptr, int badlen, suginfo_T *su, int maxcount, int banbadword, int need_cap, int interactive);
201#ifdef FEAT_EVAL
202static void spell_suggest_expr(suginfo_T *su, char_u *expr);
203#endif
204static void spell_suggest_file(suginfo_T *su, char_u *fname);
205static void spell_suggest_intern(suginfo_T *su, int interactive);
206static void spell_find_cleanup(suginfo_T *su);
207static void suggest_try_special(suginfo_T *su);
208static void suggest_try_change(suginfo_T *su);
209static void suggest_trie_walk(suginfo_T *su, langp_T *lp, char_u *fword, int soundfold);
210static void go_deeper(trystate_T *stack, int depth, int score_add);
211static void find_keepcap_word(slang_T *slang, char_u *fword, char_u *kword);
212static void score_comp_sal(suginfo_T *su);
213static void score_combine(suginfo_T *su);
214static int stp_sal_score(suggest_T *stp, suginfo_T *su, slang_T *slang, char_u *badsound);
215static void suggest_try_soundalike_prep(void);
216static void suggest_try_soundalike(suginfo_T *su);
217static void suggest_try_soundalike_finish(void);
218static void add_sound_suggest(suginfo_T *su, char_u *goodword, int score, langp_T *lp);
219static int soundfold_find(slang_T *slang, char_u *word);
220static int similar_chars(slang_T *slang, int c1, int c2);
221static void add_suggestion(suginfo_T *su, garray_T *gap, char_u *goodword, int badlen, int score, int altscore, int had_bonus, slang_T *slang, int maxsf);
222static void check_suggestions(suginfo_T *su, garray_T *gap);
223static void add_banned(suginfo_T *su, char_u *word);
224static void rescore_suggestions(suginfo_T *su);
225static void rescore_one(suginfo_T *su, suggest_T *stp);
226static int cleanup_suggestions(garray_T *gap, int maxscore, int keep);
227static int soundalike_score(char_u *goodsound, char_u *badsound);
228static int spell_edit_score(slang_T *slang, char_u *badword, char_u *goodword);
229static int spell_edit_score_limit(slang_T *slang, char_u *badword, char_u *goodword, int limit);
230static int spell_edit_score_limit_w(slang_T *slang, char_u *badword, char_u *goodword, int limit);
231
232/*
233 * Return TRUE when the sequence of flags in "compflags" plus "flag" can
234 * possibly form a valid compounded word. This also checks the COMPOUNDRULE
235 * lines if they don't contain wildcards.
236 */
237 static int
238can_be_compound(
239 trystate_T *sp,
240 slang_T *slang,
241 char_u *compflags,
242 int flag)
243{
244 // If the flag doesn't appear in sl_compstartflags or sl_compallflags
245 // then it can't possibly compound.
246 if (!byte_in_str(sp->ts_complen == sp->ts_compsplit
247 ? slang->sl_compstartflags : slang->sl_compallflags, flag))
248 return FALSE;
249
250 // If there are no wildcards, we can check if the flags collected so far
251 // possibly can form a match with COMPOUNDRULE patterns. This only
252 // makes sense when we have two or more words.
253 if (slang->sl_comprules != NULL && sp->ts_complen > sp->ts_compsplit)
254 {
255 int v;
256
257 compflags[sp->ts_complen] = flag;
258 compflags[sp->ts_complen + 1] = NUL;
259 v = match_compoundrule(slang, compflags + sp->ts_compsplit);
260 compflags[sp->ts_complen] = NUL;
261 return v;
262 }
263
264 return TRUE;
265}
266
267/*
268 * Adjust the score of common words.
269 */
270 static int
271score_wordcount_adj(
272 slang_T *slang,
273 int score,
274 char_u *word,
275 int split) // word was split, less bonus
276{
277 hashitem_T *hi;
278 wordcount_T *wc;
279 int bonus;
280 int newscore;
281
282 hi = hash_find(&slang->sl_wordcount, word);
283 if (!HASHITEM_EMPTY(hi))
284 {
285 wc = HI2WC(hi);
286 if (wc->wc_count < SCORE_THRES2)
287 bonus = SCORE_COMMON1;
288 else if (wc->wc_count < SCORE_THRES3)
289 bonus = SCORE_COMMON2;
290 else
291 bonus = SCORE_COMMON3;
292 if (split)
293 newscore = score - bonus / 2;
294 else
295 newscore = score - bonus;
296 if (newscore < 0)
297 return 0;
298 return newscore;
299 }
300 return score;
301}
302
303/*
304 * Like captype() but for a KEEPCAP word add ONECAP if the word starts with a
305 * capital. So that make_case_word() can turn WOrd into Word.
306 * Add ALLCAP for "WOrD".
307 */
308 static int
309badword_captype(char_u *word, char_u *end)
310{
311 int flags = captype(word, end);
312 int c;
313 int l, u;
314 int first;
315 char_u *p;
316
317 if (flags & WF_KEEPCAP)
318 {
319 // Count the number of UPPER and lower case letters.
320 l = u = 0;
321 first = FALSE;
322 for (p = word; p < end; MB_PTR_ADV(p))
323 {
324 c = PTR2CHAR(p);
325 if (SPELL_ISUPPER(c))
326 {
327 ++u;
328 if (p == word)
329 first = TRUE;
330 }
331 else
332 ++l;
333 }
334
335 // If there are more UPPER than lower case letters suggest an
336 // ALLCAP word. Otherwise, if the first letter is UPPER then
337 // suggest ONECAP. Exception: "ALl" most likely should be "All",
338 // require three upper case letters.
339 if (u > l && u > 2)
340 flags |= WF_ALLCAP;
341 else if (first)
342 flags |= WF_ONECAP;
343
344 if (u >= 2 && l >= 2) // maCARONI maCAroni
345 flags |= WF_MIXCAP;
346 }
347 return flags;
348}
349
350/*
351 * Opposite of offset2bytes().
352 * "pp" points to the bytes and is advanced over it.
353 * Returns the offset.
354 */
355 static int
356bytes2offset(char_u **pp)
357{
358 char_u *p = *pp;
359 int nr;
360 int c;
361
362 c = *p++;
363 if ((c & 0x80) == 0x00) // 1 byte
364 {
365 nr = c - 1;
366 }
367 else if ((c & 0xc0) == 0x80) // 2 bytes
368 {
369 nr = (c & 0x3f) - 1;
370 nr = nr * 255 + (*p++ - 1);
371 }
372 else if ((c & 0xe0) == 0xc0) // 3 bytes
373 {
374 nr = (c & 0x1f) - 1;
375 nr = nr * 255 + (*p++ - 1);
376 nr = nr * 255 + (*p++ - 1);
377 }
378 else // 4 bytes
379 {
380 nr = (c & 0x0f) - 1;
381 nr = nr * 255 + (*p++ - 1);
382 nr = nr * 255 + (*p++ - 1);
383 nr = nr * 255 + (*p++ - 1);
384 }
385
386 *pp = p;
387 return nr;
388}
389
390// values for sps_flags
391#define SPS_BEST 1
392#define SPS_FAST 2
393#define SPS_DOUBLE 4
394
395static int sps_flags = SPS_BEST; // flags from 'spellsuggest'
396static int sps_limit = 9999; // max nr of suggestions given
397
398/*
399 * Check the 'spellsuggest' option. Return FAIL if it's wrong.
400 * Sets "sps_flags" and "sps_limit".
401 */
402 int
403spell_check_sps(void)
404{
405 char_u *p;
406 char_u *s;
407 char_u buf[MAXPATHL];
408 int f;
409
410 sps_flags = 0;
411 sps_limit = 9999;
412
413 for (p = p_sps; *p != NUL; )
414 {
415 copy_option_part(&p, buf, MAXPATHL, ",");
416
417 f = 0;
418 if (VIM_ISDIGIT(*buf))
419 {
420 s = buf;
421 sps_limit = getdigits(&s);
422 if (*s != NUL && !VIM_ISDIGIT(*s))
423 f = -1;
424 }
425 else if (STRCMP(buf, "best") == 0)
426 f = SPS_BEST;
427 else if (STRCMP(buf, "fast") == 0)
428 f = SPS_FAST;
429 else if (STRCMP(buf, "double") == 0)
430 f = SPS_DOUBLE;
431 else if (STRNCMP(buf, "expr:", 5) != 0
432 && STRNCMP(buf, "file:", 5) != 0)
433 f = -1;
434
435 if (f == -1 || (sps_flags != 0 && f != 0))
436 {
437 sps_flags = SPS_BEST;
438 sps_limit = 9999;
439 return FAIL;
440 }
441 if (f != 0)
442 sps_flags = f;
443 }
444
445 if (sps_flags == 0)
446 sps_flags = SPS_BEST;
447
448 return OK;
449}
450
451/*
452 * "z=": Find badly spelled word under or after the cursor.
453 * Give suggestions for the properly spelled word.
454 * In Visual mode use the highlighted word as the bad word.
455 * When "count" is non-zero use that suggestion.
456 */
457 void
458spell_suggest(int count)
459{
460 char_u *line;
461 pos_T prev_cursor = curwin->w_cursor;
462 char_u wcopy[MAXWLEN + 2];
463 char_u *p;
464 int i;
465 int c;
466 suginfo_T sug;
467 suggest_T *stp;
468 int mouse_used;
469 int need_cap;
470 int limit;
471 int selected = count;
472 int badlen = 0;
473 int msg_scroll_save = msg_scroll;
Bram Moolenaar152e79e2020-06-10 15:32:08 +0200474 int wo_spell_save = curwin->w_p_spell;
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200475
Bram Moolenaar152e79e2020-06-10 15:32:08 +0200476 if (!curwin->w_p_spell)
477 {
478 did_set_spelllang(curwin);
479 curwin->w_p_spell = TRUE;
480 }
481
482 if (*curwin->w_s->b_p_spl == NUL)
483 {
484 emsg(_(e_no_spell));
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200485 return;
Bram Moolenaar152e79e2020-06-10 15:32:08 +0200486 }
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200487
488 if (VIsual_active)
489 {
490 // Use the Visually selected text as the bad word. But reject
491 // a multi-line selection.
492 if (curwin->w_cursor.lnum != VIsual.lnum)
493 {
494 vim_beep(BO_SPELL);
495 return;
496 }
497 badlen = (int)curwin->w_cursor.col - (int)VIsual.col;
498 if (badlen < 0)
499 badlen = -badlen;
500 else
501 curwin->w_cursor.col = VIsual.col;
502 ++badlen;
503 end_visual_mode();
504 }
505 // Find the start of the badly spelled word.
506 else if (spell_move_to(curwin, FORWARD, TRUE, TRUE, NULL) == 0
507 || curwin->w_cursor.col > prev_cursor.col)
508 {
509 // No bad word or it starts after the cursor: use the word under the
510 // cursor.
511 curwin->w_cursor = prev_cursor;
512 line = ml_get_curline();
513 p = line + curwin->w_cursor.col;
514 // Backup to before start of word.
515 while (p > line && spell_iswordp_nmw(p, curwin))
516 MB_PTR_BACK(line, p);
517 // Forward to start of word.
518 while (*p != NUL && !spell_iswordp_nmw(p, curwin))
519 MB_PTR_ADV(p);
520
521 if (!spell_iswordp_nmw(p, curwin)) // No word found.
522 {
523 beep_flush();
524 return;
525 }
526 curwin->w_cursor.col = (colnr_T)(p - line);
527 }
528
529 // Get the word and its length.
530
531 // Figure out if the word should be capitalised.
532 need_cap = check_need_cap(curwin->w_cursor.lnum, curwin->w_cursor.col);
533
534 // Make a copy of current line since autocommands may free the line.
535 line = vim_strsave(ml_get_curline());
536 if (line == NULL)
537 goto skip;
538
539 // Get the list of suggestions. Limit to 'lines' - 2 or the number in
540 // 'spellsuggest', whatever is smaller.
541 if (sps_limit > (int)Rows - 2)
542 limit = (int)Rows - 2;
543 else
544 limit = sps_limit;
545 spell_find_suggest(line + curwin->w_cursor.col, badlen, &sug, limit,
546 TRUE, need_cap, TRUE);
547
548 if (sug.su_ga.ga_len == 0)
549 msg(_("Sorry, no suggestions"));
550 else if (count > 0)
551 {
552 if (count > sug.su_ga.ga_len)
Bram Moolenaar6c52f822019-12-25 14:13:03 +0100553 smsg(_("Sorry, only %ld suggestions"), (long)sug.su_ga.ga_len);
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200554 }
555 else
556 {
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200557#ifdef FEAT_RIGHTLEFT
558 // When 'rightleft' is set the list is drawn right-left.
559 cmdmsg_rl = curwin->w_p_rl;
560 if (cmdmsg_rl)
561 msg_col = Columns - 1;
562#endif
563
564 // List the suggestions.
565 msg_start();
566 msg_row = Rows - 1; // for when 'cmdheight' > 1
567 lines_left = Rows; // avoid more prompt
568 vim_snprintf((char *)IObuff, IOSIZE, _("Change \"%.*s\" to:"),
569 sug.su_badlen, sug.su_badptr);
570#ifdef FEAT_RIGHTLEFT
571 if (cmdmsg_rl && STRNCMP(IObuff, "Change", 6) == 0)
572 {
573 // And now the rabbit from the high hat: Avoid showing the
574 // untranslated message rightleft.
575 vim_snprintf((char *)IObuff, IOSIZE, ":ot \"%.*s\" egnahC",
576 sug.su_badlen, sug.su_badptr);
577 }
578#endif
579 msg_puts((char *)IObuff);
580 msg_clr_eos();
581 msg_putchar('\n');
582
583 msg_scroll = TRUE;
584 for (i = 0; i < sug.su_ga.ga_len; ++i)
585 {
586 stp = &SUG(sug.su_ga, i);
587
588 // The suggested word may replace only part of the bad word, add
589 // the not replaced part.
590 vim_strncpy(wcopy, stp->st_word, MAXWLEN);
591 if (sug.su_badlen > stp->st_orglen)
592 vim_strncpy(wcopy + stp->st_wordlen,
593 sug.su_badptr + stp->st_orglen,
594 sug.su_badlen - stp->st_orglen);
595 vim_snprintf((char *)IObuff, IOSIZE, "%2d", i + 1);
596#ifdef FEAT_RIGHTLEFT
597 if (cmdmsg_rl)
598 rl_mirror(IObuff);
599#endif
600 msg_puts((char *)IObuff);
601
602 vim_snprintf((char *)IObuff, IOSIZE, " \"%s\"", wcopy);
603 msg_puts((char *)IObuff);
604
605 // The word may replace more than "su_badlen".
606 if (sug.su_badlen < stp->st_orglen)
607 {
608 vim_snprintf((char *)IObuff, IOSIZE, _(" < \"%.*s\""),
609 stp->st_orglen, sug.su_badptr);
610 msg_puts((char *)IObuff);
611 }
612
613 if (p_verbose > 0)
614 {
615 // Add the score.
616 if (sps_flags & (SPS_DOUBLE | SPS_BEST))
617 vim_snprintf((char *)IObuff, IOSIZE, " (%s%d - %d)",
618 stp->st_salscore ? "s " : "",
619 stp->st_score, stp->st_altscore);
620 else
621 vim_snprintf((char *)IObuff, IOSIZE, " (%d)",
622 stp->st_score);
623#ifdef FEAT_RIGHTLEFT
624 if (cmdmsg_rl)
625 // Mirror the numbers, but keep the leading space.
626 rl_mirror(IObuff + 1);
627#endif
628 msg_advance(30);
629 msg_puts((char *)IObuff);
630 }
631 msg_putchar('\n');
632 }
633
634#ifdef FEAT_RIGHTLEFT
635 cmdmsg_rl = FALSE;
636 msg_col = 0;
637#endif
638 // Ask for choice.
639 selected = prompt_for_number(&mouse_used);
640 if (mouse_used)
641 selected -= lines_left;
642 lines_left = Rows; // avoid more prompt
643 // don't delay for 'smd' in normal_cmd()
644 msg_scroll = msg_scroll_save;
645 }
646
647 if (selected > 0 && selected <= sug.su_ga.ga_len && u_save_cursor() == OK)
648 {
649 // Save the from and to text for :spellrepall.
Bram Moolenaar6c52f822019-12-25 14:13:03 +0100650 VIM_CLEAR(repl_from);
651 VIM_CLEAR(repl_to);
652
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200653 stp = &SUG(sug.su_ga, selected - 1);
654 if (sug.su_badlen > stp->st_orglen)
655 {
656 // Replacing less than "su_badlen", append the remainder to
657 // repl_to.
658 repl_from = vim_strnsave(sug.su_badptr, sug.su_badlen);
659 vim_snprintf((char *)IObuff, IOSIZE, "%s%.*s", stp->st_word,
660 sug.su_badlen - stp->st_orglen,
661 sug.su_badptr + stp->st_orglen);
662 repl_to = vim_strsave(IObuff);
663 }
664 else
665 {
666 // Replacing su_badlen or more, use the whole word.
667 repl_from = vim_strnsave(sug.su_badptr, stp->st_orglen);
668 repl_to = vim_strsave(stp->st_word);
669 }
670
671 // Replace the word.
672 p = alloc(STRLEN(line) - stp->st_orglen + stp->st_wordlen + 1);
673 if (p != NULL)
674 {
675 c = (int)(sug.su_badptr - line);
676 mch_memmove(p, line, c);
677 STRCPY(p + c, stp->st_word);
678 STRCAT(p, sug.su_badptr + stp->st_orglen);
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200679
680 // For redo we use a change-word command.
681 ResetRedobuff();
682 AppendToRedobuff((char_u *)"ciw");
683 AppendToRedobuffLit(p + c,
684 stp->st_wordlen + sug.su_badlen - stp->st_orglen);
685 AppendCharToRedobuff(ESC);
686
Bram Moolenaar6b949612020-06-29 23:18:42 +0200687 // "p" may be freed here
688 ml_replace(curwin->w_cursor.lnum, p, FALSE);
689 curwin->w_cursor.col = c;
690
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200691 changed_bytes(curwin->w_cursor.lnum, c);
692 }
693 }
694 else
695 curwin->w_cursor = prev_cursor;
696
697 spell_find_cleanup(&sug);
698skip:
699 vim_free(line);
Bram Moolenaar152e79e2020-06-10 15:32:08 +0200700 curwin->w_p_spell = wo_spell_save;
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200701}
702
703/*
704 * Find spell suggestions for "word". Return them in the growarray "*gap" as
705 * a list of allocated strings.
706 */
707 void
708spell_suggest_list(
709 garray_T *gap,
710 char_u *word,
711 int maxcount, // maximum nr of suggestions
712 int need_cap, // 'spellcapcheck' matched
713 int interactive)
714{
715 suginfo_T sug;
716 int i;
717 suggest_T *stp;
718 char_u *wcopy;
719
720 spell_find_suggest(word, 0, &sug, maxcount, FALSE, need_cap, interactive);
721
722 // Make room in "gap".
723 ga_init2(gap, sizeof(char_u *), sug.su_ga.ga_len + 1);
724 if (ga_grow(gap, sug.su_ga.ga_len) == OK)
725 {
726 for (i = 0; i < sug.su_ga.ga_len; ++i)
727 {
728 stp = &SUG(sug.su_ga, i);
729
730 // The suggested word may replace only part of "word", add the not
731 // replaced part.
732 wcopy = alloc(stp->st_wordlen
733 + (unsigned)STRLEN(sug.su_badptr + stp->st_orglen) + 1);
734 if (wcopy == NULL)
735 break;
736 STRCPY(wcopy, stp->st_word);
737 STRCPY(wcopy + stp->st_wordlen, sug.su_badptr + stp->st_orglen);
738 ((char_u **)gap->ga_data)[gap->ga_len++] = wcopy;
739 }
740 }
741
742 spell_find_cleanup(&sug);
743}
744
745/*
746 * Find spell suggestions for the word at the start of "badptr".
747 * Return the suggestions in "su->su_ga".
748 * The maximum number of suggestions is "maxcount".
749 * Note: does use info for the current window.
750 * This is based on the mechanisms of Aspell, but completely reimplemented.
751 */
752 static void
753spell_find_suggest(
754 char_u *badptr,
755 int badlen, // length of bad word or 0 if unknown
756 suginfo_T *su,
757 int maxcount,
758 int banbadword, // don't include badword in suggestions
759 int need_cap, // word should start with capital
760 int interactive)
761{
762 hlf_T attr = HLF_COUNT;
763 char_u buf[MAXPATHL];
764 char_u *p;
765 int do_combine = FALSE;
766 char_u *sps_copy;
767#ifdef FEAT_EVAL
768 static int expr_busy = FALSE;
769#endif
770 int c;
771 int i;
772 langp_T *lp;
Bram Moolenaar77a849c2021-01-20 21:42:33 +0100773 int did_intern = FALSE;
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200774
775 // Set the info in "*su".
Bram Moolenaara80faa82020-04-12 19:37:17 +0200776 CLEAR_POINTER(su);
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200777 ga_init2(&su->su_ga, (int)sizeof(suggest_T), 10);
778 ga_init2(&su->su_sga, (int)sizeof(suggest_T), 10);
779 if (*badptr == NUL)
780 return;
781 hash_init(&su->su_banned);
782
783 su->su_badptr = badptr;
784 if (badlen != 0)
785 su->su_badlen = badlen;
786 else
787 su->su_badlen = spell_check(curwin, su->su_badptr, &attr, NULL, FALSE);
788 su->su_maxcount = maxcount;
789 su->su_maxscore = SCORE_MAXINIT;
790
791 if (su->su_badlen >= MAXWLEN)
792 su->su_badlen = MAXWLEN - 1; // just in case
793 vim_strncpy(su->su_badword, su->su_badptr, su->su_badlen);
Bram Moolenaar4f135272021-06-11 19:07:40 +0200794 (void)spell_casefold(curwin, su->su_badptr, su->su_badlen,
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200795 su->su_fbadword, MAXWLEN);
796 // TODO: make this work if the case-folded text is longer than the original
797 // text. Currently an illegal byte causes wrong pointer computations.
798 su->su_fbadword[su->su_badlen] = NUL;
799
800 // get caps flags for bad word
801 su->su_badflags = badword_captype(su->su_badptr,
802 su->su_badptr + su->su_badlen);
803 if (need_cap)
804 su->su_badflags |= WF_ONECAP;
805
806 // Find the default language for sound folding. We simply use the first
807 // one in 'spelllang' that supports sound folding. That's good for when
808 // using multiple files for one language, it's not that bad when mixing
809 // languages (e.g., "pl,en").
810 for (i = 0; i < curbuf->b_s.b_langp.ga_len; ++i)
811 {
812 lp = LANGP_ENTRY(curbuf->b_s.b_langp, i);
813 if (lp->lp_sallang != NULL)
814 {
815 su->su_sallang = lp->lp_sallang;
816 break;
817 }
818 }
819
820 // Soundfold the bad word with the default sound folding, so that we don't
821 // have to do this many times.
822 if (su->su_sallang != NULL)
823 spell_soundfold(su->su_sallang, su->su_fbadword, TRUE,
824 su->su_sal_badword);
825
826 // If the word is not capitalised and spell_check() doesn't consider the
827 // word to be bad then it might need to be capitalised. Add a suggestion
828 // for that.
829 c = PTR2CHAR(su->su_badptr);
830 if (!SPELL_ISUPPER(c) && attr == HLF_COUNT)
831 {
832 make_case_word(su->su_badword, buf, WF_ONECAP);
833 add_suggestion(su, &su->su_ga, buf, su->su_badlen, SCORE_ICASE,
834 0, TRUE, su->su_sallang, FALSE);
835 }
836
837 // Ban the bad word itself. It may appear in another region.
838 if (banbadword)
839 add_banned(su, su->su_badword);
840
841 // Make a copy of 'spellsuggest', because the expression may change it.
842 sps_copy = vim_strsave(p_sps);
843 if (sps_copy == NULL)
844 return;
845
846 // Loop over the items in 'spellsuggest'.
847 for (p = sps_copy; *p != NUL; )
848 {
849 copy_option_part(&p, buf, MAXPATHL, ",");
850
851 if (STRNCMP(buf, "expr:", 5) == 0)
852 {
853#ifdef FEAT_EVAL
854 // Evaluate an expression. Skip this when called recursively,
855 // when using spellsuggest() in the expression.
856 if (!expr_busy)
857 {
858 expr_busy = TRUE;
859 spell_suggest_expr(su, buf + 5);
860 expr_busy = FALSE;
861 }
862#endif
863 }
864 else if (STRNCMP(buf, "file:", 5) == 0)
865 // Use list of suggestions in a file.
866 spell_suggest_file(su, buf + 5);
Bram Moolenaar77a849c2021-01-20 21:42:33 +0100867 else if (!did_intern)
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200868 {
Bram Moolenaar77a849c2021-01-20 21:42:33 +0100869 // Use internal method once.
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200870 spell_suggest_intern(su, interactive);
871 if (sps_flags & SPS_DOUBLE)
872 do_combine = TRUE;
Bram Moolenaar77a849c2021-01-20 21:42:33 +0100873 did_intern = TRUE;
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200874 }
875 }
876
877 vim_free(sps_copy);
878
879 if (do_combine)
880 // Combine the two list of suggestions. This must be done last,
881 // because sorting changes the order again.
882 score_combine(su);
883}
884
885#ifdef FEAT_EVAL
886/*
887 * Find suggestions by evaluating expression "expr".
888 */
889 static void
890spell_suggest_expr(suginfo_T *su, char_u *expr)
891{
892 list_T *list;
893 listitem_T *li;
894 int score;
895 char_u *p;
896
897 // The work is split up in a few parts to avoid having to export
898 // suginfo_T.
899 // First evaluate the expression and get the resulting list.
900 list = eval_spell_expr(su->su_badword, expr);
901 if (list != NULL)
902 {
903 // Loop over the items in the list.
Bram Moolenaaraeea7212020-04-02 18:50:46 +0200904 FOR_ALL_LIST_ITEMS(list, li)
Bram Moolenaar46a426c2019-09-27 12:41:56 +0200905 if (li->li_tv.v_type == VAR_LIST)
906 {
907 // Get the word and the score from the items.
908 score = get_spellword(li->li_tv.vval.v_list, &p);
909 if (score >= 0 && score <= su->su_maxscore)
910 add_suggestion(su, &su->su_ga, p, su->su_badlen,
911 score, 0, TRUE, su->su_sallang, FALSE);
912 }
913 list_unref(list);
914 }
915
916 // Remove bogus suggestions, sort and truncate at "maxcount".
917 check_suggestions(su, &su->su_ga);
918 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
919}
920#endif
921
922/*
923 * Find suggestions in file "fname". Used for "file:" in 'spellsuggest'.
924 */
925 static void
926spell_suggest_file(suginfo_T *su, char_u *fname)
927{
928 FILE *fd;
929 char_u line[MAXWLEN * 2];
930 char_u *p;
931 int len;
932 char_u cword[MAXWLEN];
933
934 // Open the file.
935 fd = mch_fopen((char *)fname, "r");
936 if (fd == NULL)
937 {
938 semsg(_(e_notopen), fname);
939 return;
940 }
941
942 // Read it line by line.
943 while (!vim_fgets(line, MAXWLEN * 2, fd) && !got_int)
944 {
945 line_breakcheck();
946
947 p = vim_strchr(line, '/');
948 if (p == NULL)
949 continue; // No Tab found, just skip the line.
950 *p++ = NUL;
951 if (STRICMP(su->su_badword, line) == 0)
952 {
953 // Match! Isolate the good word, until CR or NL.
954 for (len = 0; p[len] >= ' '; ++len)
955 ;
956 p[len] = NUL;
957
958 // If the suggestion doesn't have specific case duplicate the case
959 // of the bad word.
960 if (captype(p, NULL) == 0)
961 {
962 make_case_word(p, cword, su->su_badflags);
963 p = cword;
964 }
965
966 add_suggestion(su, &su->su_ga, p, su->su_badlen,
967 SCORE_FILE, 0, TRUE, su->su_sallang, FALSE);
968 }
969 }
970
971 fclose(fd);
972
973 // Remove bogus suggestions, sort and truncate at "maxcount".
974 check_suggestions(su, &su->su_ga);
975 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
976}
977
978/*
979 * Find suggestions for the internal method indicated by "sps_flags".
980 */
981 static void
982spell_suggest_intern(suginfo_T *su, int interactive)
983{
984 // Load the .sug file(s) that are available and not done yet.
985 suggest_load_files();
986
987 // 1. Try special cases, such as repeating a word: "the the" -> "the".
988 //
989 // Set a maximum score to limit the combination of operations that is
990 // tried.
991 suggest_try_special(su);
992
993 // 2. Try inserting/deleting/swapping/changing a letter, use REP entries
994 // from the .aff file and inserting a space (split the word).
995 suggest_try_change(su);
996
997 // For the resulting top-scorers compute the sound-a-like score.
998 if (sps_flags & SPS_DOUBLE)
999 score_comp_sal(su);
1000
1001 // 3. Try finding sound-a-like words.
1002 if ((sps_flags & SPS_FAST) == 0)
1003 {
1004 if (sps_flags & SPS_BEST)
1005 // Adjust the word score for the suggestions found so far for how
Bram Moolenaar32aa1022019-11-02 22:54:41 +01001006 // they sound like.
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001007 rescore_suggestions(su);
1008
1009 // While going through the soundfold tree "su_maxscore" is the score
1010 // for the soundfold word, limits the changes that are being tried,
1011 // and "su_sfmaxscore" the rescored score, which is set by
1012 // cleanup_suggestions().
1013 // First find words with a small edit distance, because this is much
1014 // faster and often already finds the top-N suggestions. If we didn't
1015 // find many suggestions try again with a higher edit distance.
1016 // "sl_sounddone" is used to avoid doing the same word twice.
1017 suggest_try_soundalike_prep();
1018 su->su_maxscore = SCORE_SFMAX1;
1019 su->su_sfmaxscore = SCORE_MAXINIT * 3;
1020 suggest_try_soundalike(su);
1021 if (su->su_ga.ga_len < SUG_CLEAN_COUNT(su))
1022 {
1023 // We didn't find enough matches, try again, allowing more
1024 // changes to the soundfold word.
1025 su->su_maxscore = SCORE_SFMAX2;
1026 suggest_try_soundalike(su);
1027 if (su->su_ga.ga_len < SUG_CLEAN_COUNT(su))
1028 {
1029 // Still didn't find enough matches, try again, allowing even
1030 // more changes to the soundfold word.
1031 su->su_maxscore = SCORE_SFMAX3;
1032 suggest_try_soundalike(su);
1033 }
1034 }
1035 su->su_maxscore = su->su_sfmaxscore;
1036 suggest_try_soundalike_finish();
1037 }
1038
1039 // When CTRL-C was hit while searching do show the results. Only clear
1040 // got_int when using a command, not for spellsuggest().
1041 ui_breakcheck();
1042 if (interactive && got_int)
1043 {
1044 (void)vgetc();
1045 got_int = FALSE;
1046 }
1047
1048 if ((sps_flags & SPS_DOUBLE) == 0 && su->su_ga.ga_len != 0)
1049 {
1050 if (sps_flags & SPS_BEST)
1051 // Adjust the word score for how it sounds like.
1052 rescore_suggestions(su);
1053
1054 // Remove bogus suggestions, sort and truncate at "maxcount".
1055 check_suggestions(su, &su->su_ga);
1056 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
1057 }
1058}
1059
1060/*
1061 * Free the info put in "*su" by spell_find_suggest().
1062 */
1063 static void
1064spell_find_cleanup(suginfo_T *su)
1065{
1066 int i;
1067
1068 // Free the suggestions.
1069 for (i = 0; i < su->su_ga.ga_len; ++i)
1070 vim_free(SUG(su->su_ga, i).st_word);
1071 ga_clear(&su->su_ga);
1072 for (i = 0; i < su->su_sga.ga_len; ++i)
1073 vim_free(SUG(su->su_sga, i).st_word);
1074 ga_clear(&su->su_sga);
1075
1076 // Free the banned words.
1077 hash_clear_all(&su->su_banned, 0);
1078}
1079
1080/*
1081 * Try finding suggestions by recognizing specific situations.
1082 */
1083 static void
1084suggest_try_special(suginfo_T *su)
1085{
1086 char_u *p;
1087 size_t len;
1088 int c;
1089 char_u word[MAXWLEN];
1090
1091 // Recognize a word that is repeated: "the the".
1092 p = skiptowhite(su->su_fbadword);
1093 len = p - su->su_fbadword;
1094 p = skipwhite(p);
1095 if (STRLEN(p) == len && STRNCMP(su->su_fbadword, p, len) == 0)
1096 {
1097 // Include badflags: if the badword is onecap or allcap
1098 // use that for the goodword too: "The the" -> "The".
1099 c = su->su_fbadword[len];
1100 su->su_fbadword[len] = NUL;
1101 make_case_word(su->su_fbadword, word, su->su_badflags);
1102 su->su_fbadword[len] = c;
1103
1104 // Give a soundalike score of 0, compute the score as if deleting one
1105 // character.
1106 add_suggestion(su, &su->su_ga, word, su->su_badlen,
1107 RESCORE(SCORE_REP, 0), 0, TRUE, su->su_sallang, FALSE);
1108 }
1109}
1110
1111/*
1112 * Change the 0 to 1 to measure how much time is spent in each state.
1113 * Output is dumped in "suggestprof".
1114 */
1115#if 0
1116# define SUGGEST_PROFILE
1117proftime_T current;
1118proftime_T total;
1119proftime_T times[STATE_FINAL + 1];
1120long counts[STATE_FINAL + 1];
1121
1122 static void
1123prof_init(void)
1124{
1125 for (int i = 0; i <= STATE_FINAL; ++i)
1126 {
1127 profile_zero(&times[i]);
1128 counts[i] = 0;
1129 }
1130 profile_start(&current);
1131 profile_start(&total);
1132}
1133
1134// call before changing state
1135 static void
1136prof_store(state_T state)
1137{
1138 profile_end(&current);
1139 profile_add(&times[state], &current);
1140 ++counts[state];
1141 profile_start(&current);
1142}
1143# define PROF_STORE(state) prof_store(state);
1144
1145 static void
1146prof_report(char *name)
1147{
1148 FILE *fd = fopen("suggestprof", "a");
1149
1150 profile_end(&total);
1151 fprintf(fd, "-----------------------\n");
1152 fprintf(fd, "%s: %s\n", name, profile_msg(&total));
1153 for (int i = 0; i <= STATE_FINAL; ++i)
1154 fprintf(fd, "%d: %s (%ld)\n", i, profile_msg(&times[i]), counts[i]);
1155 fclose(fd);
1156}
1157#else
1158# define PROF_STORE(state)
1159#endif
1160
1161/*
1162 * Try finding suggestions by adding/removing/swapping letters.
1163 */
1164 static void
1165suggest_try_change(suginfo_T *su)
1166{
1167 char_u fword[MAXWLEN]; // copy of the bad word, case-folded
1168 int n;
1169 char_u *p;
1170 int lpi;
1171 langp_T *lp;
1172
1173 // We make a copy of the case-folded bad word, so that we can modify it
1174 // to find matches (esp. REP items). Append some more text, changing
1175 // chars after the bad word may help.
1176 STRCPY(fword, su->su_fbadword);
1177 n = (int)STRLEN(fword);
1178 p = su->su_badptr + su->su_badlen;
Bram Moolenaar4f135272021-06-11 19:07:40 +02001179 (void)spell_casefold(curwin, p, (int)STRLEN(p), fword + n, MAXWLEN - n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001180
1181 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
1182 {
1183 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
1184
1185 // If reloading a spell file fails it's still in the list but
1186 // everything has been cleared.
1187 if (lp->lp_slang->sl_fbyts == NULL)
1188 continue;
1189
1190 // Try it for this language. Will add possible suggestions.
1191#ifdef SUGGEST_PROFILE
1192 prof_init();
1193#endif
1194 suggest_trie_walk(su, lp, fword, FALSE);
1195#ifdef SUGGEST_PROFILE
1196 prof_report("try_change");
1197#endif
1198 }
1199}
1200
1201// Check the maximum score, if we go over it we won't try this change.
1202#define TRY_DEEPER(su, stack, depth, add) \
1203 (stack[depth].ts_score + (add) < su->su_maxscore)
1204
1205/*
1206 * Try finding suggestions by adding/removing/swapping letters.
1207 *
1208 * This uses a state machine. At each node in the tree we try various
1209 * operations. When trying if an operation works "depth" is increased and the
1210 * stack[] is used to store info. This allows combinations, thus insert one
1211 * character, replace one and delete another. The number of changes is
1212 * limited by su->su_maxscore.
1213 *
1214 * After implementing this I noticed an article by Kemal Oflazer that
1215 * describes something similar: "Error-tolerant Finite State Recognition with
1216 * Applications to Morphological Analysis and Spelling Correction" (1996).
1217 * The implementation in the article is simplified and requires a stack of
1218 * unknown depth. The implementation here only needs a stack depth equal to
1219 * the length of the word.
1220 *
1221 * This is also used for the sound-folded word, "soundfold" is TRUE then.
1222 * The mechanism is the same, but we find a match with a sound-folded word
1223 * that comes from one or more original words. Each of these words may be
1224 * added, this is done by add_sound_suggest().
1225 * Don't use:
1226 * the prefix tree or the keep-case tree
1227 * "su->su_badlen"
1228 * anything to do with upper and lower case
1229 * anything to do with word or non-word characters ("spell_iswordp()")
1230 * banned words
1231 * word flags (rare, region, compounding)
1232 * word splitting for now
1233 * "similar_chars()"
1234 * use "slang->sl_repsal" instead of "lp->lp_replang->sl_rep"
1235 */
1236 static void
1237suggest_trie_walk(
1238 suginfo_T *su,
1239 langp_T *lp,
1240 char_u *fword,
1241 int soundfold)
1242{
1243 char_u tword[MAXWLEN]; // good word collected so far
1244 trystate_T stack[MAXWLEN];
1245 char_u preword[MAXWLEN * 3]; // word found with proper case;
1246 // concatenation of prefix compound
1247 // words and split word. NUL terminated
1248 // when going deeper but not when coming
1249 // back.
1250 char_u compflags[MAXWLEN]; // compound flags, one for each word
1251 trystate_T *sp;
1252 int newscore;
1253 int score;
1254 char_u *byts, *fbyts, *pbyts;
1255 idx_T *idxs, *fidxs, *pidxs;
1256 int depth;
1257 int c, c2, c3;
1258 int n = 0;
1259 int flags;
1260 garray_T *gap;
1261 idx_T arridx;
1262 int len;
1263 char_u *p;
1264 fromto_T *ftp;
1265 int fl = 0, tl;
1266 int repextra = 0; // extra bytes in fword[] from REP item
1267 slang_T *slang = lp->lp_slang;
1268 int fword_ends;
1269 int goodword_ends;
1270#ifdef DEBUG_TRIEWALK
1271 // Stores the name of the change made at each level.
1272 char_u changename[MAXWLEN][80];
1273#endif
1274 int breakcheckcount = 1000;
1275 int compound_ok;
1276
1277 // Go through the whole case-fold tree, try changes at each node.
1278 // "tword[]" contains the word collected from nodes in the tree.
1279 // "fword[]" the word we are trying to match with (initially the bad
1280 // word).
1281 depth = 0;
1282 sp = &stack[0];
Bram Moolenaara80faa82020-04-12 19:37:17 +02001283 CLEAR_POINTER(sp);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001284 sp->ts_curi = 1;
1285
1286 if (soundfold)
1287 {
1288 // Going through the soundfold tree.
1289 byts = fbyts = slang->sl_sbyts;
1290 idxs = fidxs = slang->sl_sidxs;
1291 pbyts = NULL;
1292 pidxs = NULL;
1293 sp->ts_prefixdepth = PFD_NOPREFIX;
1294 sp->ts_state = STATE_START;
1295 }
1296 else
1297 {
1298 // When there are postponed prefixes we need to use these first. At
1299 // the end of the prefix we continue in the case-fold tree.
1300 fbyts = slang->sl_fbyts;
1301 fidxs = slang->sl_fidxs;
1302 pbyts = slang->sl_pbyts;
1303 pidxs = slang->sl_pidxs;
1304 if (pbyts != NULL)
1305 {
1306 byts = pbyts;
1307 idxs = pidxs;
1308 sp->ts_prefixdepth = PFD_PREFIXTREE;
1309 sp->ts_state = STATE_NOPREFIX; // try without prefix first
1310 }
1311 else
1312 {
1313 byts = fbyts;
1314 idxs = fidxs;
1315 sp->ts_prefixdepth = PFD_NOPREFIX;
1316 sp->ts_state = STATE_START;
1317 }
1318 }
1319
1320 // Loop to find all suggestions. At each round we either:
1321 // - For the current state try one operation, advance "ts_curi",
1322 // increase "depth".
1323 // - When a state is done go to the next, set "ts_state".
1324 // - When all states are tried decrease "depth".
1325 while (depth >= 0 && !got_int)
1326 {
1327 sp = &stack[depth];
1328 switch (sp->ts_state)
1329 {
1330 case STATE_START:
1331 case STATE_NOPREFIX:
1332 // Start of node: Deal with NUL bytes, which means
1333 // tword[] may end here.
1334 arridx = sp->ts_arridx; // current node in the tree
1335 len = byts[arridx]; // bytes in this node
1336 arridx += sp->ts_curi; // index of current byte
1337
1338 if (sp->ts_prefixdepth == PFD_PREFIXTREE)
1339 {
1340 // Skip over the NUL bytes, we use them later.
1341 for (n = 0; n < len && byts[arridx + n] == 0; ++n)
1342 ;
1343 sp->ts_curi += n;
1344
1345 // Always past NUL bytes now.
1346 n = (int)sp->ts_state;
1347 PROF_STORE(sp->ts_state)
1348 sp->ts_state = STATE_ENDNUL;
1349 sp->ts_save_badflags = su->su_badflags;
1350
1351 // At end of a prefix or at start of prefixtree: check for
1352 // following word.
1353 if (byts[arridx] == 0 || n == (int)STATE_NOPREFIX)
1354 {
1355 // Set su->su_badflags to the caps type at this position.
1356 // Use the caps type until here for the prefix itself.
1357 if (has_mbyte)
1358 n = nofold_len(fword, sp->ts_fidx, su->su_badptr);
1359 else
1360 n = sp->ts_fidx;
1361 flags = badword_captype(su->su_badptr, su->su_badptr + n);
1362 su->su_badflags = badword_captype(su->su_badptr + n,
1363 su->su_badptr + su->su_badlen);
1364#ifdef DEBUG_TRIEWALK
1365 sprintf(changename[depth], "prefix");
1366#endif
1367 go_deeper(stack, depth, 0);
1368 ++depth;
1369 sp = &stack[depth];
1370 sp->ts_prefixdepth = depth - 1;
1371 byts = fbyts;
1372 idxs = fidxs;
1373 sp->ts_arridx = 0;
1374
1375 // Move the prefix to preword[] with the right case
1376 // and make find_keepcap_word() works.
1377 tword[sp->ts_twordlen] = NUL;
1378 make_case_word(tword + sp->ts_splitoff,
1379 preword + sp->ts_prewordlen, flags);
1380 sp->ts_prewordlen = (char_u)STRLEN(preword);
1381 sp->ts_splitoff = sp->ts_twordlen;
1382 }
1383 break;
1384 }
1385
1386 if (sp->ts_curi > len || byts[arridx] != 0)
1387 {
1388 // Past bytes in node and/or past NUL bytes.
1389 PROF_STORE(sp->ts_state)
1390 sp->ts_state = STATE_ENDNUL;
1391 sp->ts_save_badflags = su->su_badflags;
1392 break;
1393 }
1394
1395 // End of word in tree.
1396 ++sp->ts_curi; // eat one NUL byte
1397
1398 flags = (int)idxs[arridx];
1399
1400 // Skip words with the NOSUGGEST flag.
1401 if (flags & WF_NOSUGGEST)
1402 break;
1403
1404 fword_ends = (fword[sp->ts_fidx] == NUL
1405 || (soundfold
1406 ? VIM_ISWHITE(fword[sp->ts_fidx])
1407 : !spell_iswordp(fword + sp->ts_fidx, curwin)));
1408 tword[sp->ts_twordlen] = NUL;
1409
1410 if (sp->ts_prefixdepth <= PFD_NOTSPECIAL
Bram Moolenaar11b66002020-07-01 13:15:24 +02001411 && (sp->ts_flags & TSF_PREFIXOK) == 0
1412 && pbyts != NULL)
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001413 {
1414 // There was a prefix before the word. Check that the prefix
1415 // can be used with this word.
1416 // Count the length of the NULs in the prefix. If there are
1417 // none this must be the first try without a prefix.
1418 n = stack[sp->ts_prefixdepth].ts_arridx;
1419 len = pbyts[n++];
1420 for (c = 0; c < len && pbyts[n + c] == 0; ++c)
1421 ;
1422 if (c > 0)
1423 {
1424 c = valid_word_prefix(c, n, flags,
1425 tword + sp->ts_splitoff, slang, FALSE);
1426 if (c == 0)
1427 break;
1428
1429 // Use the WF_RARE flag for a rare prefix.
1430 if (c & WF_RAREPFX)
1431 flags |= WF_RARE;
1432
1433 // Tricky: when checking for both prefix and compounding
1434 // we run into the prefix flag first.
1435 // Remember that it's OK, so that we accept the prefix
1436 // when arriving at a compound flag.
1437 sp->ts_flags |= TSF_PREFIXOK;
1438 }
1439 }
1440
1441 // Check NEEDCOMPOUND: can't use word without compounding. Do try
1442 // appending another compound word below.
1443 if (sp->ts_complen == sp->ts_compsplit && fword_ends
1444 && (flags & WF_NEEDCOMP))
1445 goodword_ends = FALSE;
1446 else
1447 goodword_ends = TRUE;
1448
1449 p = NULL;
1450 compound_ok = TRUE;
1451 if (sp->ts_complen > sp->ts_compsplit)
1452 {
1453 if (slang->sl_nobreak)
1454 {
1455 // There was a word before this word. When there was no
1456 // change in this word (it was correct) add the first word
1457 // as a suggestion. If this word was corrected too, we
1458 // need to check if a correct word follows.
1459 if (sp->ts_fidx - sp->ts_splitfidx
1460 == sp->ts_twordlen - sp->ts_splitoff
1461 && STRNCMP(fword + sp->ts_splitfidx,
1462 tword + sp->ts_splitoff,
1463 sp->ts_fidx - sp->ts_splitfidx) == 0)
1464 {
1465 preword[sp->ts_prewordlen] = NUL;
1466 newscore = score_wordcount_adj(slang, sp->ts_score,
1467 preword + sp->ts_prewordlen,
1468 sp->ts_prewordlen > 0);
1469 // Add the suggestion if the score isn't too bad.
1470 if (newscore <= su->su_maxscore)
1471 add_suggestion(su, &su->su_ga, preword,
1472 sp->ts_splitfidx - repextra,
1473 newscore, 0, FALSE,
1474 lp->lp_sallang, FALSE);
1475 break;
1476 }
1477 }
1478 else
1479 {
1480 // There was a compound word before this word. If this
1481 // word does not support compounding then give up
1482 // (splitting is tried for the word without compound
1483 // flag).
1484 if (((unsigned)flags >> 24) == 0
1485 || sp->ts_twordlen - sp->ts_splitoff
1486 < slang->sl_compminlen)
1487 break;
1488 // For multi-byte chars check character length against
1489 // COMPOUNDMIN.
1490 if (has_mbyte
1491 && slang->sl_compminlen > 0
1492 && mb_charlen(tword + sp->ts_splitoff)
1493 < slang->sl_compminlen)
1494 break;
1495
1496 compflags[sp->ts_complen] = ((unsigned)flags >> 24);
1497 compflags[sp->ts_complen + 1] = NUL;
1498 vim_strncpy(preword + sp->ts_prewordlen,
1499 tword + sp->ts_splitoff,
1500 sp->ts_twordlen - sp->ts_splitoff);
1501
1502 // Verify CHECKCOMPOUNDPATTERN rules.
1503 if (match_checkcompoundpattern(preword, sp->ts_prewordlen,
1504 &slang->sl_comppat))
1505 compound_ok = FALSE;
1506
1507 if (compound_ok)
1508 {
1509 p = preword;
1510 while (*skiptowhite(p) != NUL)
1511 p = skipwhite(skiptowhite(p));
1512 if (fword_ends && !can_compound(slang, p,
1513 compflags + sp->ts_compsplit))
1514 // Compound is not allowed. But it may still be
1515 // possible if we add another (short) word.
1516 compound_ok = FALSE;
1517 }
1518
1519 // Get pointer to last char of previous word.
1520 p = preword + sp->ts_prewordlen;
1521 MB_PTR_BACK(preword, p);
1522 }
1523 }
1524
1525 // Form the word with proper case in preword.
1526 // If there is a word from a previous split, append.
1527 // For the soundfold tree don't change the case, simply append.
1528 if (soundfold)
1529 STRCPY(preword + sp->ts_prewordlen, tword + sp->ts_splitoff);
1530 else if (flags & WF_KEEPCAP)
1531 // Must find the word in the keep-case tree.
1532 find_keepcap_word(slang, tword + sp->ts_splitoff,
1533 preword + sp->ts_prewordlen);
1534 else
1535 {
1536 // Include badflags: If the badword is onecap or allcap
1537 // use that for the goodword too. But if the badword is
1538 // allcap and it's only one char long use onecap.
1539 c = su->su_badflags;
1540 if ((c & WF_ALLCAP)
1541 && su->su_badlen == (*mb_ptr2len)(su->su_badptr))
1542 c = WF_ONECAP;
1543 c |= flags;
1544
1545 // When appending a compound word after a word character don't
1546 // use Onecap.
1547 if (p != NULL && spell_iswordp_nmw(p, curwin))
1548 c &= ~WF_ONECAP;
1549 make_case_word(tword + sp->ts_splitoff,
1550 preword + sp->ts_prewordlen, c);
1551 }
1552
1553 if (!soundfold)
1554 {
1555 // Don't use a banned word. It may appear again as a good
1556 // word, thus remember it.
1557 if (flags & WF_BANNED)
1558 {
1559 add_banned(su, preword + sp->ts_prewordlen);
1560 break;
1561 }
1562 if ((sp->ts_complen == sp->ts_compsplit
1563 && WAS_BANNED(su, preword + sp->ts_prewordlen))
1564 || WAS_BANNED(su, preword))
1565 {
1566 if (slang->sl_compprog == NULL)
1567 break;
1568 // the word so far was banned but we may try compounding
1569 goodword_ends = FALSE;
1570 }
1571 }
1572
1573 newscore = 0;
1574 if (!soundfold) // soundfold words don't have flags
1575 {
1576 if ((flags & WF_REGION)
1577 && (((unsigned)flags >> 16) & lp->lp_region) == 0)
1578 newscore += SCORE_REGION;
1579 if (flags & WF_RARE)
1580 newscore += SCORE_RARE;
1581
1582 if (!spell_valid_case(su->su_badflags,
1583 captype(preword + sp->ts_prewordlen, NULL)))
1584 newscore += SCORE_ICASE;
1585 }
1586
1587 // TODO: how about splitting in the soundfold tree?
1588 if (fword_ends
1589 && goodword_ends
1590 && sp->ts_fidx >= sp->ts_fidxtry
1591 && compound_ok)
1592 {
1593 // The badword also ends: add suggestions.
1594#ifdef DEBUG_TRIEWALK
1595 if (soundfold && STRCMP(preword, "smwrd") == 0)
1596 {
1597 int j;
1598
1599 // print the stack of changes that brought us here
1600 smsg("------ %s -------", fword);
1601 for (j = 0; j < depth; ++j)
1602 smsg("%s", changename[j]);
1603 }
1604#endif
1605 if (soundfold)
1606 {
1607 // For soundfolded words we need to find the original
1608 // words, the edit distance and then add them.
1609 add_sound_suggest(su, preword, sp->ts_score, lp);
1610 }
1611 else if (sp->ts_fidx > 0)
1612 {
1613 // Give a penalty when changing non-word char to word
1614 // char, e.g., "thes," -> "these".
1615 p = fword + sp->ts_fidx;
1616 MB_PTR_BACK(fword, p);
1617 if (!spell_iswordp(p, curwin))
1618 {
1619 p = preword + STRLEN(preword);
1620 MB_PTR_BACK(preword, p);
1621 if (spell_iswordp(p, curwin))
1622 newscore += SCORE_NONWORD;
1623 }
1624
1625 // Give a bonus to words seen before.
1626 score = score_wordcount_adj(slang,
1627 sp->ts_score + newscore,
1628 preword + sp->ts_prewordlen,
1629 sp->ts_prewordlen > 0);
1630
1631 // Add the suggestion if the score isn't too bad.
1632 if (score <= su->su_maxscore)
1633 {
1634 add_suggestion(su, &su->su_ga, preword,
1635 sp->ts_fidx - repextra,
1636 score, 0, FALSE, lp->lp_sallang, FALSE);
1637
1638 if (su->su_badflags & WF_MIXCAP)
1639 {
1640 // We really don't know if the word should be
1641 // upper or lower case, add both.
1642 c = captype(preword, NULL);
1643 if (c == 0 || c == WF_ALLCAP)
1644 {
1645 make_case_word(tword + sp->ts_splitoff,
1646 preword + sp->ts_prewordlen,
1647 c == 0 ? WF_ALLCAP : 0);
1648
1649 add_suggestion(su, &su->su_ga, preword,
1650 sp->ts_fidx - repextra,
1651 score + SCORE_ICASE, 0, FALSE,
1652 lp->lp_sallang, FALSE);
1653 }
1654 }
1655 }
1656 }
1657 }
1658
1659 // Try word split and/or compounding.
1660 if ((sp->ts_fidx >= sp->ts_fidxtry || fword_ends)
1661 // Don't split halfway a character.
1662 && (!has_mbyte || sp->ts_tcharlen == 0))
1663 {
1664 int try_compound;
1665 int try_split;
1666
1667 // If past the end of the bad word don't try a split.
1668 // Otherwise try changing the next word. E.g., find
1669 // suggestions for "the the" where the second "the" is
1670 // different. It's done like a split.
1671 // TODO: word split for soundfold words
1672 try_split = (sp->ts_fidx - repextra < su->su_badlen)
1673 && !soundfold;
1674
1675 // Get here in several situations:
1676 // 1. The word in the tree ends:
1677 // If the word allows compounding try that. Otherwise try
1678 // a split by inserting a space. For both check that a
1679 // valid words starts at fword[sp->ts_fidx].
1680 // For NOBREAK do like compounding to be able to check if
1681 // the next word is valid.
1682 // 2. The badword does end, but it was due to a change (e.g.,
1683 // a swap). No need to split, but do check that the
1684 // following word is valid.
1685 // 3. The badword and the word in the tree end. It may still
1686 // be possible to compound another (short) word.
1687 try_compound = FALSE;
1688 if (!soundfold
1689 && !slang->sl_nocompoundsugs
1690 && slang->sl_compprog != NULL
1691 && ((unsigned)flags >> 24) != 0
1692 && sp->ts_twordlen - sp->ts_splitoff
1693 >= slang->sl_compminlen
1694 && (!has_mbyte
1695 || slang->sl_compminlen == 0
1696 || mb_charlen(tword + sp->ts_splitoff)
1697 >= slang->sl_compminlen)
1698 && (slang->sl_compsylmax < MAXWLEN
1699 || sp->ts_complen + 1 - sp->ts_compsplit
1700 < slang->sl_compmax)
1701 && (can_be_compound(sp, slang,
1702 compflags, ((unsigned)flags >> 24))))
1703
1704 {
1705 try_compound = TRUE;
1706 compflags[sp->ts_complen] = ((unsigned)flags >> 24);
1707 compflags[sp->ts_complen + 1] = NUL;
1708 }
1709
1710 // For NOBREAK we never try splitting, it won't make any word
1711 // valid.
1712 if (slang->sl_nobreak && !slang->sl_nocompoundsugs)
1713 try_compound = TRUE;
1714
1715 // If we could add a compound word, and it's also possible to
1716 // split at this point, do the split first and set
1717 // TSF_DIDSPLIT to avoid doing it again.
1718 else if (!fword_ends
1719 && try_compound
1720 && (sp->ts_flags & TSF_DIDSPLIT) == 0)
1721 {
1722 try_compound = FALSE;
1723 sp->ts_flags |= TSF_DIDSPLIT;
1724 --sp->ts_curi; // do the same NUL again
1725 compflags[sp->ts_complen] = NUL;
1726 }
1727 else
1728 sp->ts_flags &= ~TSF_DIDSPLIT;
1729
1730 if (try_split || try_compound)
1731 {
1732 if (!try_compound && (!fword_ends || !goodword_ends))
1733 {
1734 // If we're going to split need to check that the
1735 // words so far are valid for compounding. If there
1736 // is only one word it must not have the NEEDCOMPOUND
1737 // flag.
1738 if (sp->ts_complen == sp->ts_compsplit
1739 && (flags & WF_NEEDCOMP))
1740 break;
1741 p = preword;
1742 while (*skiptowhite(p) != NUL)
1743 p = skipwhite(skiptowhite(p));
1744 if (sp->ts_complen > sp->ts_compsplit
1745 && !can_compound(slang, p,
1746 compflags + sp->ts_compsplit))
1747 break;
1748
1749 if (slang->sl_nosplitsugs)
1750 newscore += SCORE_SPLIT_NO;
1751 else
1752 newscore += SCORE_SPLIT;
1753
1754 // Give a bonus to words seen before.
1755 newscore = score_wordcount_adj(slang, newscore,
1756 preword + sp->ts_prewordlen, TRUE);
1757 }
1758
1759 if (TRY_DEEPER(su, stack, depth, newscore))
1760 {
1761 go_deeper(stack, depth, newscore);
1762#ifdef DEBUG_TRIEWALK
1763 if (!try_compound && !fword_ends)
1764 sprintf(changename[depth], "%.*s-%s: split",
1765 sp->ts_twordlen, tword, fword + sp->ts_fidx);
1766 else
1767 sprintf(changename[depth], "%.*s-%s: compound",
1768 sp->ts_twordlen, tword, fword + sp->ts_fidx);
1769#endif
1770 // Save things to be restored at STATE_SPLITUNDO.
1771 sp->ts_save_badflags = su->su_badflags;
1772 PROF_STORE(sp->ts_state)
1773 sp->ts_state = STATE_SPLITUNDO;
1774
1775 ++depth;
1776 sp = &stack[depth];
1777
1778 // Append a space to preword when splitting.
1779 if (!try_compound && !fword_ends)
1780 STRCAT(preword, " ");
1781 sp->ts_prewordlen = (char_u)STRLEN(preword);
1782 sp->ts_splitoff = sp->ts_twordlen;
1783 sp->ts_splitfidx = sp->ts_fidx;
1784
1785 // If the badword has a non-word character at this
1786 // position skip it. That means replacing the
1787 // non-word character with a space. Always skip a
1788 // character when the word ends. But only when the
1789 // good word can end.
1790 if (((!try_compound && !spell_iswordp_nmw(fword
1791 + sp->ts_fidx,
1792 curwin))
1793 || fword_ends)
1794 && fword[sp->ts_fidx] != NUL
1795 && goodword_ends)
1796 {
1797 int l;
1798
Bram Moolenaar1614a142019-10-06 22:00:13 +02001799 l = mb_ptr2len(fword + sp->ts_fidx);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001800 if (fword_ends)
1801 {
1802 // Copy the skipped character to preword.
1803 mch_memmove(preword + sp->ts_prewordlen,
1804 fword + sp->ts_fidx, l);
1805 sp->ts_prewordlen += l;
1806 preword[sp->ts_prewordlen] = NUL;
1807 }
1808 else
1809 sp->ts_score -= SCORE_SPLIT - SCORE_SUBST;
1810 sp->ts_fidx += l;
1811 }
1812
1813 // When compounding include compound flag in
1814 // compflags[] (already set above). When splitting we
1815 // may start compounding over again.
1816 if (try_compound)
1817 ++sp->ts_complen;
1818 else
1819 sp->ts_compsplit = sp->ts_complen;
1820 sp->ts_prefixdepth = PFD_NOPREFIX;
1821
1822 // set su->su_badflags to the caps type at this
1823 // position
1824 if (has_mbyte)
1825 n = nofold_len(fword, sp->ts_fidx, su->su_badptr);
1826 else
1827 n = sp->ts_fidx;
1828 su->su_badflags = badword_captype(su->su_badptr + n,
1829 su->su_badptr + su->su_badlen);
1830
1831 // Restart at top of the tree.
1832 sp->ts_arridx = 0;
1833
1834 // If there are postponed prefixes, try these too.
1835 if (pbyts != NULL)
1836 {
1837 byts = pbyts;
1838 idxs = pidxs;
1839 sp->ts_prefixdepth = PFD_PREFIXTREE;
1840 PROF_STORE(sp->ts_state)
1841 sp->ts_state = STATE_NOPREFIX;
1842 }
1843 }
1844 }
1845 }
1846 break;
1847
1848 case STATE_SPLITUNDO:
1849 // Undo the changes done for word split or compound word.
1850 su->su_badflags = sp->ts_save_badflags;
1851
1852 // Continue looking for NUL bytes.
1853 PROF_STORE(sp->ts_state)
1854 sp->ts_state = STATE_START;
1855
1856 // In case we went into the prefix tree.
1857 byts = fbyts;
1858 idxs = fidxs;
1859 break;
1860
1861 case STATE_ENDNUL:
1862 // Past the NUL bytes in the node.
1863 su->su_badflags = sp->ts_save_badflags;
1864 if (fword[sp->ts_fidx] == NUL && sp->ts_tcharlen == 0)
1865 {
1866 // The badword ends, can't use STATE_PLAIN.
1867 PROF_STORE(sp->ts_state)
1868 sp->ts_state = STATE_DEL;
1869 break;
1870 }
1871 PROF_STORE(sp->ts_state)
1872 sp->ts_state = STATE_PLAIN;
1873 // FALLTHROUGH
1874
1875 case STATE_PLAIN:
1876 // Go over all possible bytes at this node, add each to tword[]
1877 // and use child node. "ts_curi" is the index.
1878 arridx = sp->ts_arridx;
1879 if (sp->ts_curi > byts[arridx])
1880 {
1881 // Done all bytes at this node, do next state. When still at
1882 // already changed bytes skip the other tricks.
1883 PROF_STORE(sp->ts_state)
1884 if (sp->ts_fidx >= sp->ts_fidxtry)
1885 sp->ts_state = STATE_DEL;
1886 else
1887 sp->ts_state = STATE_FINAL;
1888 }
1889 else
1890 {
1891 arridx += sp->ts_curi++;
1892 c = byts[arridx];
1893
1894 // Normal byte, go one level deeper. If it's not equal to the
1895 // byte in the bad word adjust the score. But don't even try
1896 // when the byte was already changed. And don't try when we
1897 // just deleted this byte, accepting it is always cheaper than
1898 // delete + substitute.
1899 if (c == fword[sp->ts_fidx]
1900 || (sp->ts_tcharlen > 0 && sp->ts_isdiff != DIFF_NONE))
1901 newscore = 0;
1902 else
1903 newscore = SCORE_SUBST;
1904 if ((newscore == 0
1905 || (sp->ts_fidx >= sp->ts_fidxtry
1906 && ((sp->ts_flags & TSF_DIDDEL) == 0
1907 || c != fword[sp->ts_delidx])))
1908 && TRY_DEEPER(su, stack, depth, newscore))
1909 {
1910 go_deeper(stack, depth, newscore);
1911#ifdef DEBUG_TRIEWALK
1912 if (newscore > 0)
1913 sprintf(changename[depth], "%.*s-%s: subst %c to %c",
1914 sp->ts_twordlen, tword, fword + sp->ts_fidx,
1915 fword[sp->ts_fidx], c);
1916 else
1917 sprintf(changename[depth], "%.*s-%s: accept %c",
1918 sp->ts_twordlen, tword, fword + sp->ts_fidx,
1919 fword[sp->ts_fidx]);
1920#endif
1921 ++depth;
1922 sp = &stack[depth];
1923 ++sp->ts_fidx;
1924 tword[sp->ts_twordlen++] = c;
1925 sp->ts_arridx = idxs[arridx];
1926 if (newscore == SCORE_SUBST)
1927 sp->ts_isdiff = DIFF_YES;
1928 if (has_mbyte)
1929 {
1930 // Multi-byte characters are a bit complicated to
1931 // handle: They differ when any of the bytes differ
1932 // and then their length may also differ.
1933 if (sp->ts_tcharlen == 0)
1934 {
1935 // First byte.
1936 sp->ts_tcharidx = 0;
1937 sp->ts_tcharlen = MB_BYTE2LEN(c);
1938 sp->ts_fcharstart = sp->ts_fidx - 1;
1939 sp->ts_isdiff = (newscore != 0)
1940 ? DIFF_YES : DIFF_NONE;
1941 }
1942 else if (sp->ts_isdiff == DIFF_INSERT)
1943 // When inserting trail bytes don't advance in the
1944 // bad word.
1945 --sp->ts_fidx;
1946 if (++sp->ts_tcharidx == sp->ts_tcharlen)
1947 {
1948 // Last byte of character.
1949 if (sp->ts_isdiff == DIFF_YES)
1950 {
1951 // Correct ts_fidx for the byte length of the
1952 // character (we didn't check that before).
1953 sp->ts_fidx = sp->ts_fcharstart
Bram Moolenaar1614a142019-10-06 22:00:13 +02001954 + mb_ptr2len(
Bram Moolenaar46a426c2019-09-27 12:41:56 +02001955 fword + sp->ts_fcharstart);
1956 // For changing a composing character adjust
1957 // the score from SCORE_SUBST to
1958 // SCORE_SUBCOMP.
1959 if (enc_utf8
1960 && utf_iscomposing(
1961 utf_ptr2char(tword
1962 + sp->ts_twordlen
1963 - sp->ts_tcharlen))
1964 && utf_iscomposing(
1965 utf_ptr2char(fword
1966 + sp->ts_fcharstart)))
1967 sp->ts_score -=
1968 SCORE_SUBST - SCORE_SUBCOMP;
1969
1970 // For a similar character adjust score from
1971 // SCORE_SUBST to SCORE_SIMILAR.
1972 else if (!soundfold
1973 && slang->sl_has_map
1974 && similar_chars(slang,
1975 mb_ptr2char(tword
1976 + sp->ts_twordlen
1977 - sp->ts_tcharlen),
1978 mb_ptr2char(fword
1979 + sp->ts_fcharstart)))
1980 sp->ts_score -=
1981 SCORE_SUBST - SCORE_SIMILAR;
1982 }
1983 else if (sp->ts_isdiff == DIFF_INSERT
1984 && sp->ts_twordlen > sp->ts_tcharlen)
1985 {
1986 p = tword + sp->ts_twordlen - sp->ts_tcharlen;
1987 c = mb_ptr2char(p);
1988 if (enc_utf8 && utf_iscomposing(c))
1989 {
1990 // Inserting a composing char doesn't
1991 // count that much.
1992 sp->ts_score -= SCORE_INS - SCORE_INSCOMP;
1993 }
1994 else
1995 {
1996 // If the previous character was the same,
1997 // thus doubling a character, give a bonus
1998 // to the score. Also for the soundfold
1999 // tree (might seem illogical but does
2000 // give better scores).
2001 MB_PTR_BACK(tword, p);
2002 if (c == mb_ptr2char(p))
2003 sp->ts_score -= SCORE_INS
2004 - SCORE_INSDUP;
2005 }
2006 }
2007
2008 // Starting a new char, reset the length.
2009 sp->ts_tcharlen = 0;
2010 }
2011 }
2012 else
2013 {
2014 // If we found a similar char adjust the score.
2015 // We do this after calling go_deeper() because
2016 // it's slow.
2017 if (newscore != 0
2018 && !soundfold
2019 && slang->sl_has_map
2020 && similar_chars(slang,
2021 c, fword[sp->ts_fidx - 1]))
2022 sp->ts_score -= SCORE_SUBST - SCORE_SIMILAR;
2023 }
2024 }
2025 }
2026 break;
2027
2028 case STATE_DEL:
2029 // When past the first byte of a multi-byte char don't try
2030 // delete/insert/swap a character.
2031 if (has_mbyte && sp->ts_tcharlen > 0)
2032 {
2033 PROF_STORE(sp->ts_state)
2034 sp->ts_state = STATE_FINAL;
2035 break;
2036 }
2037 // Try skipping one character in the bad word (delete it).
2038 PROF_STORE(sp->ts_state)
2039 sp->ts_state = STATE_INS_PREP;
2040 sp->ts_curi = 1;
2041 if (soundfold && sp->ts_fidx == 0 && fword[sp->ts_fidx] == '*')
2042 // Deleting a vowel at the start of a word counts less, see
2043 // soundalike_score().
2044 newscore = 2 * SCORE_DEL / 3;
2045 else
2046 newscore = SCORE_DEL;
2047 if (fword[sp->ts_fidx] != NUL
2048 && TRY_DEEPER(su, stack, depth, newscore))
2049 {
2050 go_deeper(stack, depth, newscore);
2051#ifdef DEBUG_TRIEWALK
2052 sprintf(changename[depth], "%.*s-%s: delete %c",
2053 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2054 fword[sp->ts_fidx]);
2055#endif
2056 ++depth;
2057
2058 // Remember what character we deleted, so that we can avoid
2059 // inserting it again.
2060 stack[depth].ts_flags |= TSF_DIDDEL;
2061 stack[depth].ts_delidx = sp->ts_fidx;
2062
2063 // Advance over the character in fword[]. Give a bonus to the
2064 // score if the same character is following "nn" -> "n". It's
2065 // a bit illogical for soundfold tree but it does give better
2066 // results.
2067 if (has_mbyte)
2068 {
2069 c = mb_ptr2char(fword + sp->ts_fidx);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002070 stack[depth].ts_fidx += mb_ptr2len(fword + sp->ts_fidx);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002071 if (enc_utf8 && utf_iscomposing(c))
2072 stack[depth].ts_score -= SCORE_DEL - SCORE_DELCOMP;
2073 else if (c == mb_ptr2char(fword + stack[depth].ts_fidx))
2074 stack[depth].ts_score -= SCORE_DEL - SCORE_DELDUP;
2075 }
2076 else
2077 {
2078 ++stack[depth].ts_fidx;
2079 if (fword[sp->ts_fidx] == fword[sp->ts_fidx + 1])
2080 stack[depth].ts_score -= SCORE_DEL - SCORE_DELDUP;
2081 }
2082 break;
2083 }
2084 // FALLTHROUGH
2085
2086 case STATE_INS_PREP:
2087 if (sp->ts_flags & TSF_DIDDEL)
2088 {
2089 // If we just deleted a byte then inserting won't make sense,
2090 // a substitute is always cheaper.
2091 PROF_STORE(sp->ts_state)
2092 sp->ts_state = STATE_SWAP;
2093 break;
2094 }
2095
2096 // skip over NUL bytes
2097 n = sp->ts_arridx;
2098 for (;;)
2099 {
2100 if (sp->ts_curi > byts[n])
2101 {
2102 // Only NUL bytes at this node, go to next state.
2103 PROF_STORE(sp->ts_state)
2104 sp->ts_state = STATE_SWAP;
2105 break;
2106 }
2107 if (byts[n + sp->ts_curi] != NUL)
2108 {
2109 // Found a byte to insert.
2110 PROF_STORE(sp->ts_state)
2111 sp->ts_state = STATE_INS;
2112 break;
2113 }
2114 ++sp->ts_curi;
2115 }
2116 break;
2117
2118 // FALLTHROUGH
2119
2120 case STATE_INS:
2121 // Insert one byte. Repeat this for each possible byte at this
2122 // node.
2123 n = sp->ts_arridx;
2124 if (sp->ts_curi > byts[n])
2125 {
2126 // Done all bytes at this node, go to next state.
2127 PROF_STORE(sp->ts_state)
2128 sp->ts_state = STATE_SWAP;
2129 break;
2130 }
2131
2132 // Do one more byte at this node, but:
2133 // - Skip NUL bytes.
2134 // - Skip the byte if it's equal to the byte in the word,
2135 // accepting that byte is always better.
2136 n += sp->ts_curi++;
2137 c = byts[n];
2138 if (soundfold && sp->ts_twordlen == 0 && c == '*')
2139 // Inserting a vowel at the start of a word counts less,
2140 // see soundalike_score().
2141 newscore = 2 * SCORE_INS / 3;
2142 else
2143 newscore = SCORE_INS;
2144 if (c != fword[sp->ts_fidx]
2145 && TRY_DEEPER(su, stack, depth, newscore))
2146 {
2147 go_deeper(stack, depth, newscore);
2148#ifdef DEBUG_TRIEWALK
2149 sprintf(changename[depth], "%.*s-%s: insert %c",
2150 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2151 c);
2152#endif
2153 ++depth;
2154 sp = &stack[depth];
2155 tword[sp->ts_twordlen++] = c;
2156 sp->ts_arridx = idxs[n];
2157 if (has_mbyte)
2158 {
2159 fl = MB_BYTE2LEN(c);
2160 if (fl > 1)
2161 {
2162 // There are following bytes for the same character.
2163 // We must find all bytes before trying
2164 // delete/insert/swap/etc.
2165 sp->ts_tcharlen = fl;
2166 sp->ts_tcharidx = 1;
2167 sp->ts_isdiff = DIFF_INSERT;
2168 }
2169 }
2170 else
2171 fl = 1;
2172 if (fl == 1)
2173 {
2174 // If the previous character was the same, thus doubling a
2175 // character, give a bonus to the score. Also for
2176 // soundfold words (illogical but does give a better
2177 // score).
2178 if (sp->ts_twordlen >= 2
2179 && tword[sp->ts_twordlen - 2] == c)
2180 sp->ts_score -= SCORE_INS - SCORE_INSDUP;
2181 }
2182 }
2183 break;
2184
2185 case STATE_SWAP:
2186 // Swap two bytes in the bad word: "12" -> "21".
2187 // We change "fword" here, it's changed back afterwards at
2188 // STATE_UNSWAP.
2189 p = fword + sp->ts_fidx;
2190 c = *p;
2191 if (c == NUL)
2192 {
2193 // End of word, can't swap or replace.
2194 PROF_STORE(sp->ts_state)
2195 sp->ts_state = STATE_FINAL;
2196 break;
2197 }
2198
2199 // Don't swap if the first character is not a word character.
2200 // SWAP3 etc. also don't make sense then.
2201 if (!soundfold && !spell_iswordp(p, curwin))
2202 {
2203 PROF_STORE(sp->ts_state)
2204 sp->ts_state = STATE_REP_INI;
2205 break;
2206 }
2207
2208 if (has_mbyte)
2209 {
2210 n = MB_CPTR2LEN(p);
2211 c = mb_ptr2char(p);
2212 if (p[n] == NUL)
2213 c2 = NUL;
2214 else if (!soundfold && !spell_iswordp(p + n, curwin))
2215 c2 = c; // don't swap non-word char
2216 else
2217 c2 = mb_ptr2char(p + n);
2218 }
2219 else
2220 {
2221 if (p[1] == NUL)
2222 c2 = NUL;
2223 else if (!soundfold && !spell_iswordp(p + 1, curwin))
2224 c2 = c; // don't swap non-word char
2225 else
2226 c2 = p[1];
2227 }
2228
2229 // When the second character is NUL we can't swap.
2230 if (c2 == NUL)
2231 {
2232 PROF_STORE(sp->ts_state)
2233 sp->ts_state = STATE_REP_INI;
2234 break;
2235 }
2236
2237 // When characters are identical, swap won't do anything.
2238 // Also get here if the second char is not a word character.
2239 if (c == c2)
2240 {
2241 PROF_STORE(sp->ts_state)
2242 sp->ts_state = STATE_SWAP3;
2243 break;
2244 }
2245 if (c2 != NUL && TRY_DEEPER(su, stack, depth, SCORE_SWAP))
2246 {
2247 go_deeper(stack, depth, SCORE_SWAP);
2248#ifdef DEBUG_TRIEWALK
2249 sprintf(changename[depth], "%.*s-%s: swap %c and %c",
2250 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2251 c, c2);
2252#endif
2253 PROF_STORE(sp->ts_state)
2254 sp->ts_state = STATE_UNSWAP;
2255 ++depth;
2256 if (has_mbyte)
2257 {
2258 fl = mb_char2len(c2);
2259 mch_memmove(p, p + n, fl);
2260 mb_char2bytes(c, p + fl);
2261 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl;
2262 }
2263 else
2264 {
2265 p[0] = c2;
2266 p[1] = c;
2267 stack[depth].ts_fidxtry = sp->ts_fidx + 2;
2268 }
2269 }
2270 else
2271 {
2272 // If this swap doesn't work then SWAP3 won't either.
2273 PROF_STORE(sp->ts_state)
2274 sp->ts_state = STATE_REP_INI;
2275 }
2276 break;
2277
2278 case STATE_UNSWAP:
2279 // Undo the STATE_SWAP swap: "21" -> "12".
2280 p = fword + sp->ts_fidx;
2281 if (has_mbyte)
2282 {
Bram Moolenaar1614a142019-10-06 22:00:13 +02002283 n = mb_ptr2len(p);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002284 c = mb_ptr2char(p + n);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002285 mch_memmove(p + mb_ptr2len(p + n), p, n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002286 mb_char2bytes(c, p);
2287 }
2288 else
2289 {
2290 c = *p;
2291 *p = p[1];
2292 p[1] = c;
2293 }
2294 // FALLTHROUGH
2295
2296 case STATE_SWAP3:
2297 // Swap two bytes, skipping one: "123" -> "321". We change
2298 // "fword" here, it's changed back afterwards at STATE_UNSWAP3.
2299 p = fword + sp->ts_fidx;
2300 if (has_mbyte)
2301 {
2302 n = MB_CPTR2LEN(p);
2303 c = mb_ptr2char(p);
2304 fl = MB_CPTR2LEN(p + n);
2305 c2 = mb_ptr2char(p + n);
2306 if (!soundfold && !spell_iswordp(p + n + fl, curwin))
2307 c3 = c; // don't swap non-word char
2308 else
2309 c3 = mb_ptr2char(p + n + fl);
2310 }
2311 else
2312 {
2313 c = *p;
2314 c2 = p[1];
2315 if (!soundfold && !spell_iswordp(p + 2, curwin))
2316 c3 = c; // don't swap non-word char
2317 else
2318 c3 = p[2];
2319 }
2320
2321 // When characters are identical: "121" then SWAP3 result is
2322 // identical, ROT3L result is same as SWAP: "211", ROT3L result is
2323 // same as SWAP on next char: "112". Thus skip all swapping.
2324 // Also skip when c3 is NUL.
2325 // Also get here when the third character is not a word character.
2326 // Second character may any char: "a.b" -> "b.a"
2327 if (c == c3 || c3 == NUL)
2328 {
2329 PROF_STORE(sp->ts_state)
2330 sp->ts_state = STATE_REP_INI;
2331 break;
2332 }
2333 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3))
2334 {
2335 go_deeper(stack, depth, SCORE_SWAP3);
2336#ifdef DEBUG_TRIEWALK
2337 sprintf(changename[depth], "%.*s-%s: swap3 %c and %c",
2338 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2339 c, c3);
2340#endif
2341 PROF_STORE(sp->ts_state)
2342 sp->ts_state = STATE_UNSWAP3;
2343 ++depth;
2344 if (has_mbyte)
2345 {
2346 tl = mb_char2len(c3);
2347 mch_memmove(p, p + n + fl, tl);
2348 mb_char2bytes(c2, p + tl);
2349 mb_char2bytes(c, p + fl + tl);
2350 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl + tl;
2351 }
2352 else
2353 {
2354 p[0] = p[2];
2355 p[2] = c;
2356 stack[depth].ts_fidxtry = sp->ts_fidx + 3;
2357 }
2358 }
2359 else
2360 {
2361 PROF_STORE(sp->ts_state)
2362 sp->ts_state = STATE_REP_INI;
2363 }
2364 break;
2365
2366 case STATE_UNSWAP3:
2367 // Undo STATE_SWAP3: "321" -> "123"
2368 p = fword + sp->ts_fidx;
2369 if (has_mbyte)
2370 {
Bram Moolenaar1614a142019-10-06 22:00:13 +02002371 n = mb_ptr2len(p);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002372 c2 = mb_ptr2char(p + n);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002373 fl = mb_ptr2len(p + n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002374 c = mb_ptr2char(p + n + fl);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002375 tl = mb_ptr2len(p + n + fl);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002376 mch_memmove(p + fl + tl, p, n);
2377 mb_char2bytes(c, p);
2378 mb_char2bytes(c2, p + tl);
2379 p = p + tl;
2380 }
2381 else
2382 {
2383 c = *p;
2384 *p = p[2];
2385 p[2] = c;
2386 ++p;
2387 }
2388
2389 if (!soundfold && !spell_iswordp(p, curwin))
2390 {
2391 // Middle char is not a word char, skip the rotate. First and
2392 // third char were already checked at swap and swap3.
2393 PROF_STORE(sp->ts_state)
2394 sp->ts_state = STATE_REP_INI;
2395 break;
2396 }
2397
2398 // Rotate three characters left: "123" -> "231". We change
2399 // "fword" here, it's changed back afterwards at STATE_UNROT3L.
2400 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3))
2401 {
2402 go_deeper(stack, depth, SCORE_SWAP3);
2403#ifdef DEBUG_TRIEWALK
2404 p = fword + sp->ts_fidx;
2405 sprintf(changename[depth], "%.*s-%s: rotate left %c%c%c",
2406 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2407 p[0], p[1], p[2]);
2408#endif
2409 PROF_STORE(sp->ts_state)
2410 sp->ts_state = STATE_UNROT3L;
2411 ++depth;
2412 p = fword + sp->ts_fidx;
2413 if (has_mbyte)
2414 {
2415 n = MB_CPTR2LEN(p);
2416 c = mb_ptr2char(p);
2417 fl = MB_CPTR2LEN(p + n);
2418 fl += MB_CPTR2LEN(p + n + fl);
2419 mch_memmove(p, p + n, fl);
2420 mb_char2bytes(c, p + fl);
2421 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl;
2422 }
2423 else
2424 {
2425 c = *p;
2426 *p = p[1];
2427 p[1] = p[2];
2428 p[2] = c;
2429 stack[depth].ts_fidxtry = sp->ts_fidx + 3;
2430 }
2431 }
2432 else
2433 {
2434 PROF_STORE(sp->ts_state)
2435 sp->ts_state = STATE_REP_INI;
2436 }
2437 break;
2438
2439 case STATE_UNROT3L:
2440 // Undo ROT3L: "231" -> "123"
2441 p = fword + sp->ts_fidx;
2442 if (has_mbyte)
2443 {
Bram Moolenaar1614a142019-10-06 22:00:13 +02002444 n = mb_ptr2len(p);
2445 n += mb_ptr2len(p + n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002446 c = mb_ptr2char(p + n);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002447 tl = mb_ptr2len(p + n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002448 mch_memmove(p + tl, p, n);
2449 mb_char2bytes(c, p);
2450 }
2451 else
2452 {
2453 c = p[2];
2454 p[2] = p[1];
2455 p[1] = *p;
2456 *p = c;
2457 }
2458
2459 // Rotate three bytes right: "123" -> "312". We change "fword"
2460 // here, it's changed back afterwards at STATE_UNROT3R.
2461 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3))
2462 {
2463 go_deeper(stack, depth, SCORE_SWAP3);
2464#ifdef DEBUG_TRIEWALK
2465 p = fword + sp->ts_fidx;
2466 sprintf(changename[depth], "%.*s-%s: rotate right %c%c%c",
2467 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2468 p[0], p[1], p[2]);
2469#endif
2470 PROF_STORE(sp->ts_state)
2471 sp->ts_state = STATE_UNROT3R;
2472 ++depth;
2473 p = fword + sp->ts_fidx;
2474 if (has_mbyte)
2475 {
2476 n = MB_CPTR2LEN(p);
2477 n += MB_CPTR2LEN(p + n);
2478 c = mb_ptr2char(p + n);
2479 tl = MB_CPTR2LEN(p + n);
2480 mch_memmove(p + tl, p, n);
2481 mb_char2bytes(c, p);
2482 stack[depth].ts_fidxtry = sp->ts_fidx + n + tl;
2483 }
2484 else
2485 {
2486 c = p[2];
2487 p[2] = p[1];
2488 p[1] = *p;
2489 *p = c;
2490 stack[depth].ts_fidxtry = sp->ts_fidx + 3;
2491 }
2492 }
2493 else
2494 {
2495 PROF_STORE(sp->ts_state)
2496 sp->ts_state = STATE_REP_INI;
2497 }
2498 break;
2499
2500 case STATE_UNROT3R:
2501 // Undo ROT3R: "312" -> "123"
2502 p = fword + sp->ts_fidx;
2503 if (has_mbyte)
2504 {
2505 c = mb_ptr2char(p);
Bram Moolenaar1614a142019-10-06 22:00:13 +02002506 tl = mb_ptr2len(p);
2507 n = mb_ptr2len(p + tl);
2508 n += mb_ptr2len(p + tl + n);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02002509 mch_memmove(p, p + tl, n);
2510 mb_char2bytes(c, p + n);
2511 }
2512 else
2513 {
2514 c = *p;
2515 *p = p[1];
2516 p[1] = p[2];
2517 p[2] = c;
2518 }
2519 // FALLTHROUGH
2520
2521 case STATE_REP_INI:
2522 // Check if matching with REP items from the .aff file would work.
2523 // Quickly skip if:
2524 // - there are no REP items and we are not in the soundfold trie
2525 // - the score is going to be too high anyway
2526 // - already applied a REP item or swapped here
2527 if ((lp->lp_replang == NULL && !soundfold)
2528 || sp->ts_score + SCORE_REP >= su->su_maxscore
2529 || sp->ts_fidx < sp->ts_fidxtry)
2530 {
2531 PROF_STORE(sp->ts_state)
2532 sp->ts_state = STATE_FINAL;
2533 break;
2534 }
2535
2536 // Use the first byte to quickly find the first entry that may
2537 // match. If the index is -1 there is none.
2538 if (soundfold)
2539 sp->ts_curi = slang->sl_repsal_first[fword[sp->ts_fidx]];
2540 else
2541 sp->ts_curi = lp->lp_replang->sl_rep_first[fword[sp->ts_fidx]];
2542
2543 if (sp->ts_curi < 0)
2544 {
2545 PROF_STORE(sp->ts_state)
2546 sp->ts_state = STATE_FINAL;
2547 break;
2548 }
2549
2550 PROF_STORE(sp->ts_state)
2551 sp->ts_state = STATE_REP;
2552 // FALLTHROUGH
2553
2554 case STATE_REP:
2555 // Try matching with REP items from the .aff file. For each match
2556 // replace the characters and check if the resulting word is
2557 // valid.
2558 p = fword + sp->ts_fidx;
2559
2560 if (soundfold)
2561 gap = &slang->sl_repsal;
2562 else
2563 gap = &lp->lp_replang->sl_rep;
2564 while (sp->ts_curi < gap->ga_len)
2565 {
2566 ftp = (fromto_T *)gap->ga_data + sp->ts_curi++;
2567 if (*ftp->ft_from != *p)
2568 {
2569 // past possible matching entries
2570 sp->ts_curi = gap->ga_len;
2571 break;
2572 }
2573 if (STRNCMP(ftp->ft_from, p, STRLEN(ftp->ft_from)) == 0
2574 && TRY_DEEPER(su, stack, depth, SCORE_REP))
2575 {
2576 go_deeper(stack, depth, SCORE_REP);
2577#ifdef DEBUG_TRIEWALK
2578 sprintf(changename[depth], "%.*s-%s: replace %s with %s",
2579 sp->ts_twordlen, tword, fword + sp->ts_fidx,
2580 ftp->ft_from, ftp->ft_to);
2581#endif
2582 // Need to undo this afterwards.
2583 PROF_STORE(sp->ts_state)
2584 sp->ts_state = STATE_REP_UNDO;
2585
2586 // Change the "from" to the "to" string.
2587 ++depth;
2588 fl = (int)STRLEN(ftp->ft_from);
2589 tl = (int)STRLEN(ftp->ft_to);
2590 if (fl != tl)
2591 {
2592 STRMOVE(p + tl, p + fl);
2593 repextra += tl - fl;
2594 }
2595 mch_memmove(p, ftp->ft_to, tl);
2596 stack[depth].ts_fidxtry = sp->ts_fidx + tl;
2597 stack[depth].ts_tcharlen = 0;
2598 break;
2599 }
2600 }
2601
2602 if (sp->ts_curi >= gap->ga_len && sp->ts_state == STATE_REP)
2603 {
2604 // No (more) matches.
2605 PROF_STORE(sp->ts_state)
2606 sp->ts_state = STATE_FINAL;
2607 }
2608
2609 break;
2610
2611 case STATE_REP_UNDO:
2612 // Undo a REP replacement and continue with the next one.
2613 if (soundfold)
2614 gap = &slang->sl_repsal;
2615 else
2616 gap = &lp->lp_replang->sl_rep;
2617 ftp = (fromto_T *)gap->ga_data + sp->ts_curi - 1;
2618 fl = (int)STRLEN(ftp->ft_from);
2619 tl = (int)STRLEN(ftp->ft_to);
2620 p = fword + sp->ts_fidx;
2621 if (fl != tl)
2622 {
2623 STRMOVE(p + fl, p + tl);
2624 repextra -= tl - fl;
2625 }
2626 mch_memmove(p, ftp->ft_from, fl);
2627 PROF_STORE(sp->ts_state)
2628 sp->ts_state = STATE_REP;
2629 break;
2630
2631 default:
2632 // Did all possible states at this level, go up one level.
2633 --depth;
2634
2635 if (depth >= 0 && stack[depth].ts_prefixdepth == PFD_PREFIXTREE)
2636 {
2637 // Continue in or go back to the prefix tree.
2638 byts = pbyts;
2639 idxs = pidxs;
2640 }
2641
2642 // Don't check for CTRL-C too often, it takes time.
2643 if (--breakcheckcount == 0)
2644 {
2645 ui_breakcheck();
2646 breakcheckcount = 1000;
2647 }
2648 }
2649 }
2650}
2651
2652
2653/*
2654 * Go one level deeper in the tree.
2655 */
2656 static void
2657go_deeper(trystate_T *stack, int depth, int score_add)
2658{
2659 stack[depth + 1] = stack[depth];
2660 stack[depth + 1].ts_state = STATE_START;
2661 stack[depth + 1].ts_score = stack[depth].ts_score + score_add;
2662 stack[depth + 1].ts_curi = 1; // start just after length byte
2663 stack[depth + 1].ts_flags = 0;
2664}
2665
2666/*
2667 * "fword" is a good word with case folded. Find the matching keep-case
2668 * words and put it in "kword".
2669 * Theoretically there could be several keep-case words that result in the
2670 * same case-folded word, but we only find one...
2671 */
2672 static void
2673find_keepcap_word(slang_T *slang, char_u *fword, char_u *kword)
2674{
2675 char_u uword[MAXWLEN]; // "fword" in upper-case
2676 int depth;
2677 idx_T tryidx;
2678
2679 // The following arrays are used at each depth in the tree.
2680 idx_T arridx[MAXWLEN];
2681 int round[MAXWLEN];
2682 int fwordidx[MAXWLEN];
2683 int uwordidx[MAXWLEN];
2684 int kwordlen[MAXWLEN];
2685
2686 int flen, ulen;
2687 int l;
2688 int len;
2689 int c;
2690 idx_T lo, hi, m;
2691 char_u *p;
2692 char_u *byts = slang->sl_kbyts; // array with bytes of the words
2693 idx_T *idxs = slang->sl_kidxs; // array with indexes
2694
2695 if (byts == NULL)
2696 {
2697 // array is empty: "cannot happen"
2698 *kword = NUL;
2699 return;
2700 }
2701
2702 // Make an all-cap version of "fword".
2703 allcap_copy(fword, uword);
2704
2705 // Each character needs to be tried both case-folded and upper-case.
2706 // All this gets very complicated if we keep in mind that changing case
2707 // may change the byte length of a multi-byte character...
2708 depth = 0;
2709 arridx[0] = 0;
2710 round[0] = 0;
2711 fwordidx[0] = 0;
2712 uwordidx[0] = 0;
2713 kwordlen[0] = 0;
2714 while (depth >= 0)
2715 {
2716 if (fword[fwordidx[depth]] == NUL)
2717 {
2718 // We are at the end of "fword". If the tree allows a word to end
2719 // here we have found a match.
2720 if (byts[arridx[depth] + 1] == 0)
2721 {
2722 kword[kwordlen[depth]] = NUL;
2723 return;
2724 }
2725
2726 // kword is getting too long, continue one level up
2727 --depth;
2728 }
2729 else if (++round[depth] > 2)
2730 {
2731 // tried both fold-case and upper-case character, continue one
2732 // level up
2733 --depth;
2734 }
2735 else
2736 {
2737 // round[depth] == 1: Try using the folded-case character.
2738 // round[depth] == 2: Try using the upper-case character.
2739 if (has_mbyte)
2740 {
2741 flen = MB_CPTR2LEN(fword + fwordidx[depth]);
2742 ulen = MB_CPTR2LEN(uword + uwordidx[depth]);
2743 }
2744 else
2745 ulen = flen = 1;
2746 if (round[depth] == 1)
2747 {
2748 p = fword + fwordidx[depth];
2749 l = flen;
2750 }
2751 else
2752 {
2753 p = uword + uwordidx[depth];
2754 l = ulen;
2755 }
2756
2757 for (tryidx = arridx[depth]; l > 0; --l)
2758 {
2759 // Perform a binary search in the list of accepted bytes.
2760 len = byts[tryidx++];
2761 c = *p++;
2762 lo = tryidx;
2763 hi = tryidx + len - 1;
2764 while (lo < hi)
2765 {
2766 m = (lo + hi) / 2;
2767 if (byts[m] > c)
2768 hi = m - 1;
2769 else if (byts[m] < c)
2770 lo = m + 1;
2771 else
2772 {
2773 lo = hi = m;
2774 break;
2775 }
2776 }
2777
2778 // Stop if there is no matching byte.
2779 if (hi < lo || byts[lo] != c)
2780 break;
2781
2782 // Continue at the child (if there is one).
2783 tryidx = idxs[lo];
2784 }
2785
2786 if (l == 0)
2787 {
2788 // Found the matching char. Copy it to "kword" and go a
2789 // level deeper.
2790 if (round[depth] == 1)
2791 {
2792 STRNCPY(kword + kwordlen[depth], fword + fwordidx[depth],
2793 flen);
2794 kwordlen[depth + 1] = kwordlen[depth] + flen;
2795 }
2796 else
2797 {
2798 STRNCPY(kword + kwordlen[depth], uword + uwordidx[depth],
2799 ulen);
2800 kwordlen[depth + 1] = kwordlen[depth] + ulen;
2801 }
2802 fwordidx[depth + 1] = fwordidx[depth] + flen;
2803 uwordidx[depth + 1] = uwordidx[depth] + ulen;
2804
2805 ++depth;
2806 arridx[depth] = tryidx;
2807 round[depth] = 0;
2808 }
2809 }
2810 }
2811
2812 // Didn't find it: "cannot happen".
2813 *kword = NUL;
2814}
2815
2816/*
2817 * Compute the sound-a-like score for suggestions in su->su_ga and add them to
2818 * su->su_sga.
2819 */
2820 static void
2821score_comp_sal(suginfo_T *su)
2822{
2823 langp_T *lp;
2824 char_u badsound[MAXWLEN];
2825 int i;
2826 suggest_T *stp;
2827 suggest_T *sstp;
2828 int score;
2829 int lpi;
2830
2831 if (ga_grow(&su->su_sga, su->su_ga.ga_len) == FAIL)
2832 return;
2833
2834 // Use the sound-folding of the first language that supports it.
2835 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
2836 {
2837 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
2838 if (lp->lp_slang->sl_sal.ga_len > 0)
2839 {
2840 // soundfold the bad word
2841 spell_soundfold(lp->lp_slang, su->su_fbadword, TRUE, badsound);
2842
2843 for (i = 0; i < su->su_ga.ga_len; ++i)
2844 {
2845 stp = &SUG(su->su_ga, i);
2846
2847 // Case-fold the suggested word, sound-fold it and compute the
2848 // sound-a-like score.
2849 score = stp_sal_score(stp, su, lp->lp_slang, badsound);
2850 if (score < SCORE_MAXMAX)
2851 {
2852 // Add the suggestion.
2853 sstp = &SUG(su->su_sga, su->su_sga.ga_len);
2854 sstp->st_word = vim_strsave(stp->st_word);
2855 if (sstp->st_word != NULL)
2856 {
2857 sstp->st_wordlen = stp->st_wordlen;
2858 sstp->st_score = score;
2859 sstp->st_altscore = 0;
2860 sstp->st_orglen = stp->st_orglen;
2861 ++su->su_sga.ga_len;
2862 }
2863 }
2864 }
2865 break;
2866 }
2867 }
2868}
2869
2870/*
2871 * Combine the list of suggestions in su->su_ga and su->su_sga.
2872 * They are entwined.
2873 */
2874 static void
2875score_combine(suginfo_T *su)
2876{
2877 int i;
2878 int j;
2879 garray_T ga;
2880 garray_T *gap;
2881 langp_T *lp;
2882 suggest_T *stp;
2883 char_u *p;
2884 char_u badsound[MAXWLEN];
2885 int round;
2886 int lpi;
2887 slang_T *slang = NULL;
2888
2889 // Add the alternate score to su_ga.
2890 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
2891 {
2892 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
2893 if (lp->lp_slang->sl_sal.ga_len > 0)
2894 {
2895 // soundfold the bad word
2896 slang = lp->lp_slang;
2897 spell_soundfold(slang, su->su_fbadword, TRUE, badsound);
2898
2899 for (i = 0; i < su->su_ga.ga_len; ++i)
2900 {
2901 stp = &SUG(su->su_ga, i);
2902 stp->st_altscore = stp_sal_score(stp, su, slang, badsound);
2903 if (stp->st_altscore == SCORE_MAXMAX)
2904 stp->st_score = (stp->st_score * 3 + SCORE_BIG) / 4;
2905 else
2906 stp->st_score = (stp->st_score * 3
2907 + stp->st_altscore) / 4;
2908 stp->st_salscore = FALSE;
2909 }
2910 break;
2911 }
2912 }
2913
2914 if (slang == NULL) // Using "double" without sound folding.
2915 {
2916 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore,
2917 su->su_maxcount);
2918 return;
2919 }
2920
2921 // Add the alternate score to su_sga.
2922 for (i = 0; i < su->su_sga.ga_len; ++i)
2923 {
2924 stp = &SUG(su->su_sga, i);
2925 stp->st_altscore = spell_edit_score(slang,
2926 su->su_badword, stp->st_word);
2927 if (stp->st_score == SCORE_MAXMAX)
2928 stp->st_score = (SCORE_BIG * 7 + stp->st_altscore) / 8;
2929 else
2930 stp->st_score = (stp->st_score * 7 + stp->st_altscore) / 8;
2931 stp->st_salscore = TRUE;
2932 }
2933
2934 // Remove bad suggestions, sort the suggestions and truncate at "maxcount"
2935 // for both lists.
2936 check_suggestions(su, &su->su_ga);
2937 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
2938 check_suggestions(su, &su->su_sga);
2939 (void)cleanup_suggestions(&su->su_sga, su->su_maxscore, su->su_maxcount);
2940
2941 ga_init2(&ga, (int)sizeof(suginfo_T), 1);
2942 if (ga_grow(&ga, su->su_ga.ga_len + su->su_sga.ga_len) == FAIL)
2943 return;
2944
2945 stp = &SUG(ga, 0);
2946 for (i = 0; i < su->su_ga.ga_len || i < su->su_sga.ga_len; ++i)
2947 {
2948 // round 1: get a suggestion from su_ga
2949 // round 2: get a suggestion from su_sga
2950 for (round = 1; round <= 2; ++round)
2951 {
2952 gap = round == 1 ? &su->su_ga : &su->su_sga;
2953 if (i < gap->ga_len)
2954 {
2955 // Don't add a word if it's already there.
2956 p = SUG(*gap, i).st_word;
2957 for (j = 0; j < ga.ga_len; ++j)
2958 if (STRCMP(stp[j].st_word, p) == 0)
2959 break;
2960 if (j == ga.ga_len)
2961 stp[ga.ga_len++] = SUG(*gap, i);
2962 else
2963 vim_free(p);
2964 }
2965 }
2966 }
2967
2968 ga_clear(&su->su_ga);
2969 ga_clear(&su->su_sga);
2970
2971 // Truncate the list to the number of suggestions that will be displayed.
2972 if (ga.ga_len > su->su_maxcount)
2973 {
2974 for (i = su->su_maxcount; i < ga.ga_len; ++i)
2975 vim_free(stp[i].st_word);
2976 ga.ga_len = su->su_maxcount;
2977 }
2978
2979 su->su_ga = ga;
2980}
2981
2982/*
2983 * For the goodword in "stp" compute the soundalike score compared to the
2984 * badword.
2985 */
2986 static int
2987stp_sal_score(
2988 suggest_T *stp,
2989 suginfo_T *su,
2990 slang_T *slang,
2991 char_u *badsound) // sound-folded badword
2992{
2993 char_u *p;
2994 char_u *pbad;
2995 char_u *pgood;
2996 char_u badsound2[MAXWLEN];
2997 char_u fword[MAXWLEN];
2998 char_u goodsound[MAXWLEN];
2999 char_u goodword[MAXWLEN];
3000 int lendiff;
3001
3002 lendiff = (int)(su->su_badlen - stp->st_orglen);
3003 if (lendiff >= 0)
3004 pbad = badsound;
3005 else
3006 {
3007 // soundfold the bad word with more characters following
Bram Moolenaar4f135272021-06-11 19:07:40 +02003008 (void)spell_casefold(curwin,
3009 su->su_badptr, stp->st_orglen, fword, MAXWLEN);
Bram Moolenaar46a426c2019-09-27 12:41:56 +02003010
3011 // When joining two words the sound often changes a lot. E.g., "t he"
3012 // sounds like "t h" while "the" sounds like "@". Avoid that by
3013 // removing the space. Don't do it when the good word also contains a
3014 // space.
3015 if (VIM_ISWHITE(su->su_badptr[su->su_badlen])
3016 && *skiptowhite(stp->st_word) == NUL)
3017 for (p = fword; *(p = skiptowhite(p)) != NUL; )
3018 STRMOVE(p, p + 1);
3019
3020 spell_soundfold(slang, fword, TRUE, badsound2);
3021 pbad = badsound2;
3022 }
3023
3024 if (lendiff > 0 && stp->st_wordlen + lendiff < MAXWLEN)
3025 {
3026 // Add part of the bad word to the good word, so that we soundfold
3027 // what replaces the bad word.
3028 STRCPY(goodword, stp->st_word);
3029 vim_strncpy(goodword + stp->st_wordlen,
3030 su->su_badptr + su->su_badlen - lendiff, lendiff);
3031 pgood = goodword;
3032 }
3033 else
3034 pgood = stp->st_word;
3035
3036 // Sound-fold the word and compute the score for the difference.
3037 spell_soundfold(slang, pgood, FALSE, goodsound);
3038
3039 return soundalike_score(goodsound, pbad);
3040}
3041
3042// structure used to store soundfolded words that add_sound_suggest() has
3043// handled already.
3044typedef struct
3045{
3046 short sft_score; // lowest score used
3047 char_u sft_word[1]; // soundfolded word, actually longer
3048} sftword_T;
3049
3050static sftword_T dumsft;
3051#define HIKEY2SFT(p) ((sftword_T *)(p - (dumsft.sft_word - (char_u *)&dumsft)))
3052#define HI2SFT(hi) HIKEY2SFT((hi)->hi_key)
3053
3054/*
3055 * Prepare for calling suggest_try_soundalike().
3056 */
3057 static void
3058suggest_try_soundalike_prep(void)
3059{
3060 langp_T *lp;
3061 int lpi;
3062 slang_T *slang;
3063
3064 // Do this for all languages that support sound folding and for which a
3065 // .sug file has been loaded.
3066 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
3067 {
3068 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
3069 slang = lp->lp_slang;
3070 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL)
3071 // prepare the hashtable used by add_sound_suggest()
3072 hash_init(&slang->sl_sounddone);
3073 }
3074}
3075
3076/*
3077 * Find suggestions by comparing the word in a sound-a-like form.
3078 * Note: This doesn't support postponed prefixes.
3079 */
3080 static void
3081suggest_try_soundalike(suginfo_T *su)
3082{
3083 char_u salword[MAXWLEN];
3084 langp_T *lp;
3085 int lpi;
3086 slang_T *slang;
3087
3088 // Do this for all languages that support sound folding and for which a
3089 // .sug file has been loaded.
3090 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
3091 {
3092 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
3093 slang = lp->lp_slang;
3094 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL)
3095 {
3096 // soundfold the bad word
3097 spell_soundfold(slang, su->su_fbadword, TRUE, salword);
3098
3099 // try all kinds of inserts/deletes/swaps/etc.
3100 // TODO: also soundfold the next words, so that we can try joining
3101 // and splitting
3102#ifdef SUGGEST_PROFILE
3103 prof_init();
3104#endif
3105 suggest_trie_walk(su, lp, salword, TRUE);
3106#ifdef SUGGEST_PROFILE
3107 prof_report("soundalike");
3108#endif
3109 }
3110 }
3111}
3112
3113/*
3114 * Finish up after calling suggest_try_soundalike().
3115 */
3116 static void
3117suggest_try_soundalike_finish(void)
3118{
3119 langp_T *lp;
3120 int lpi;
3121 slang_T *slang;
3122 int todo;
3123 hashitem_T *hi;
3124
3125 // Do this for all languages that support sound folding and for which a
3126 // .sug file has been loaded.
3127 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
3128 {
3129 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
3130 slang = lp->lp_slang;
3131 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL)
3132 {
3133 // Free the info about handled words.
3134 todo = (int)slang->sl_sounddone.ht_used;
3135 for (hi = slang->sl_sounddone.ht_array; todo > 0; ++hi)
3136 if (!HASHITEM_EMPTY(hi))
3137 {
3138 vim_free(HI2SFT(hi));
3139 --todo;
3140 }
3141
3142 // Clear the hashtable, it may also be used by another region.
3143 hash_clear(&slang->sl_sounddone);
3144 hash_init(&slang->sl_sounddone);
3145 }
3146 }
3147}
3148
3149/*
3150 * A match with a soundfolded word is found. Add the good word(s) that
3151 * produce this soundfolded word.
3152 */
3153 static void
3154add_sound_suggest(
3155 suginfo_T *su,
3156 char_u *goodword,
3157 int score, // soundfold score
3158 langp_T *lp)
3159{
3160 slang_T *slang = lp->lp_slang; // language for sound folding
3161 int sfwordnr;
3162 char_u *nrline;
3163 int orgnr;
3164 char_u theword[MAXWLEN];
3165 int i;
3166 int wlen;
3167 char_u *byts;
3168 idx_T *idxs;
3169 int n;
3170 int wordcount;
3171 int wc;
3172 int goodscore;
3173 hash_T hash;
3174 hashitem_T *hi;
3175 sftword_T *sft;
3176 int bc, gc;
3177 int limit;
3178
3179 // It's very well possible that the same soundfold word is found several
3180 // times with different scores. Since the following is quite slow only do
3181 // the words that have a better score than before. Use a hashtable to
3182 // remember the words that have been done.
3183 hash = hash_hash(goodword);
3184 hi = hash_lookup(&slang->sl_sounddone, goodword, hash);
3185 if (HASHITEM_EMPTY(hi))
3186 {
3187 sft = alloc(sizeof(sftword_T) + STRLEN(goodword));
3188 if (sft != NULL)
3189 {
3190 sft->sft_score = score;
3191 STRCPY(sft->sft_word, goodword);
3192 hash_add_item(&slang->sl_sounddone, hi, sft->sft_word, hash);
3193 }
3194 }
3195 else
3196 {
3197 sft = HI2SFT(hi);
3198 if (score >= sft->sft_score)
3199 return;
3200 sft->sft_score = score;
3201 }
3202
3203 // Find the word nr in the soundfold tree.
3204 sfwordnr = soundfold_find(slang, goodword);
3205 if (sfwordnr < 0)
3206 {
3207 internal_error("add_sound_suggest()");
3208 return;
3209 }
3210
3211 // go over the list of good words that produce this soundfold word
3212 nrline = ml_get_buf(slang->sl_sugbuf, (linenr_T)(sfwordnr + 1), FALSE);
3213 orgnr = 0;
3214 while (*nrline != NUL)
3215 {
3216 // The wordnr was stored in a minimal nr of bytes as an offset to the
3217 // previous wordnr.
3218 orgnr += bytes2offset(&nrline);
3219
3220 byts = slang->sl_fbyts;
3221 idxs = slang->sl_fidxs;
3222
3223 // Lookup the word "orgnr" one of the two tries.
3224 n = 0;
3225 wordcount = 0;
3226 for (wlen = 0; wlen < MAXWLEN - 3; ++wlen)
3227 {
3228 i = 1;
3229 if (wordcount == orgnr && byts[n + 1] == NUL)
3230 break; // found end of word
3231
3232 if (byts[n + 1] == NUL)
3233 ++wordcount;
3234
3235 // skip over the NUL bytes
3236 for ( ; byts[n + i] == NUL; ++i)
3237 if (i > byts[n]) // safety check
3238 {
3239 STRCPY(theword + wlen, "BAD");
3240 wlen += 3;
3241 goto badword;
3242 }
3243
3244 // One of the siblings must have the word.
3245 for ( ; i < byts[n]; ++i)
3246 {
3247 wc = idxs[idxs[n + i]]; // nr of words under this byte
3248 if (wordcount + wc > orgnr)
3249 break;
3250 wordcount += wc;
3251 }
3252
3253 theword[wlen] = byts[n + i];
3254 n = idxs[n + i];
3255 }
3256badword:
3257 theword[wlen] = NUL;
3258
3259 // Go over the possible flags and regions.
3260 for (; i <= byts[n] && byts[n + i] == NUL; ++i)
3261 {
3262 char_u cword[MAXWLEN];
3263 char_u *p;
3264 int flags = (int)idxs[n + i];
3265
3266 // Skip words with the NOSUGGEST flag
3267 if (flags & WF_NOSUGGEST)
3268 continue;
3269
3270 if (flags & WF_KEEPCAP)
3271 {
3272 // Must find the word in the keep-case tree.
3273 find_keepcap_word(slang, theword, cword);
3274 p = cword;
3275 }
3276 else
3277 {
3278 flags |= su->su_badflags;
3279 if ((flags & WF_CAPMASK) != 0)
3280 {
3281 // Need to fix case according to "flags".
3282 make_case_word(theword, cword, flags);
3283 p = cword;
3284 }
3285 else
3286 p = theword;
3287 }
3288
3289 // Add the suggestion.
3290 if (sps_flags & SPS_DOUBLE)
3291 {
3292 // Add the suggestion if the score isn't too bad.
3293 if (score <= su->su_maxscore)
3294 add_suggestion(su, &su->su_sga, p, su->su_badlen,
3295 score, 0, FALSE, slang, FALSE);
3296 }
3297 else
3298 {
3299 // Add a penalty for words in another region.
3300 if ((flags & WF_REGION)
3301 && (((unsigned)flags >> 16) & lp->lp_region) == 0)
3302 goodscore = SCORE_REGION;
3303 else
3304 goodscore = 0;
3305
3306 // Add a small penalty for changing the first letter from
3307 // lower to upper case. Helps for "tath" -> "Kath", which is
3308 // less common than "tath" -> "path". Don't do it when the
3309 // letter is the same, that has already been counted.
3310 gc = PTR2CHAR(p);
3311 if (SPELL_ISUPPER(gc))
3312 {
3313 bc = PTR2CHAR(su->su_badword);
3314 if (!SPELL_ISUPPER(bc)
3315 && SPELL_TOFOLD(bc) != SPELL_TOFOLD(gc))
3316 goodscore += SCORE_ICASE / 2;
3317 }
3318
3319 // Compute the score for the good word. This only does letter
3320 // insert/delete/swap/replace. REP items are not considered,
3321 // which may make the score a bit higher.
3322 // Use a limit for the score to make it work faster. Use
3323 // MAXSCORE(), because RESCORE() will change the score.
3324 // If the limit is very high then the iterative method is
3325 // inefficient, using an array is quicker.
3326 limit = MAXSCORE(su->su_sfmaxscore - goodscore, score);
3327 if (limit > SCORE_LIMITMAX)
3328 goodscore += spell_edit_score(slang, su->su_badword, p);
3329 else
3330 goodscore += spell_edit_score_limit(slang, su->su_badword,
3331 p, limit);
3332
3333 // When going over the limit don't bother to do the rest.
3334 if (goodscore < SCORE_MAXMAX)
3335 {
3336 // Give a bonus to words seen before.
3337 goodscore = score_wordcount_adj(slang, goodscore, p, FALSE);
3338
3339 // Add the suggestion if the score isn't too bad.
3340 goodscore = RESCORE(goodscore, score);
3341 if (goodscore <= su->su_sfmaxscore)
3342 add_suggestion(su, &su->su_ga, p, su->su_badlen,
3343 goodscore, score, TRUE, slang, TRUE);
3344 }
3345 }
3346 }
3347 // smsg("word %s (%d): %s (%d)", sftword, sftnr, theword, orgnr);
3348 }
3349}
3350
3351/*
3352 * Find word "word" in fold-case tree for "slang" and return the word number.
3353 */
3354 static int
3355soundfold_find(slang_T *slang, char_u *word)
3356{
3357 idx_T arridx = 0;
3358 int len;
3359 int wlen = 0;
3360 int c;
3361 char_u *ptr = word;
3362 char_u *byts;
3363 idx_T *idxs;
3364 int wordnr = 0;
3365
3366 byts = slang->sl_sbyts;
3367 idxs = slang->sl_sidxs;
3368
3369 for (;;)
3370 {
3371 // First byte is the number of possible bytes.
3372 len = byts[arridx++];
3373
3374 // If the first possible byte is a zero the word could end here.
3375 // If the word ends we found the word. If not skip the NUL bytes.
3376 c = ptr[wlen];
3377 if (byts[arridx] == NUL)
3378 {
3379 if (c == NUL)
3380 break;
3381
3382 // Skip over the zeros, there can be several.
3383 while (len > 0 && byts[arridx] == NUL)
3384 {
3385 ++arridx;
3386 --len;
3387 }
3388 if (len == 0)
3389 return -1; // no children, word should have ended here
3390 ++wordnr;
3391 }
3392
3393 // If the word ends we didn't find it.
3394 if (c == NUL)
3395 return -1;
3396
3397 // Perform a binary search in the list of accepted bytes.
3398 if (c == TAB) // <Tab> is handled like <Space>
3399 c = ' ';
3400 while (byts[arridx] < c)
3401 {
3402 // The word count is in the first idxs[] entry of the child.
3403 wordnr += idxs[idxs[arridx]];
3404 ++arridx;
3405 if (--len == 0) // end of the bytes, didn't find it
3406 return -1;
3407 }
3408 if (byts[arridx] != c) // didn't find the byte
3409 return -1;
3410
3411 // Continue at the child (if there is one).
3412 arridx = idxs[arridx];
3413 ++wlen;
3414
3415 // One space in the good word may stand for several spaces in the
3416 // checked word.
3417 if (c == ' ')
3418 while (ptr[wlen] == ' ' || ptr[wlen] == TAB)
3419 ++wlen;
3420 }
3421
3422 return wordnr;
3423}
3424
3425/*
3426 * Return TRUE if "c1" and "c2" are similar characters according to the MAP
3427 * lines in the .aff file.
3428 */
3429 static int
3430similar_chars(slang_T *slang, int c1, int c2)
3431{
3432 int m1, m2;
3433 char_u buf[MB_MAXBYTES + 1];
3434 hashitem_T *hi;
3435
3436 if (c1 >= 256)
3437 {
3438 buf[mb_char2bytes(c1, buf)] = 0;
3439 hi = hash_find(&slang->sl_map_hash, buf);
3440 if (HASHITEM_EMPTY(hi))
3441 m1 = 0;
3442 else
3443 m1 = mb_ptr2char(hi->hi_key + STRLEN(hi->hi_key) + 1);
3444 }
3445 else
3446 m1 = slang->sl_map_array[c1];
3447 if (m1 == 0)
3448 return FALSE;
3449
3450
3451 if (c2 >= 256)
3452 {
3453 buf[mb_char2bytes(c2, buf)] = 0;
3454 hi = hash_find(&slang->sl_map_hash, buf);
3455 if (HASHITEM_EMPTY(hi))
3456 m2 = 0;
3457 else
3458 m2 = mb_ptr2char(hi->hi_key + STRLEN(hi->hi_key) + 1);
3459 }
3460 else
3461 m2 = slang->sl_map_array[c2];
3462
3463 return m1 == m2;
3464}
3465
3466/*
3467 * Add a suggestion to the list of suggestions.
3468 * For a suggestion that is already in the list the lowest score is remembered.
3469 */
3470 static void
3471add_suggestion(
3472 suginfo_T *su,
3473 garray_T *gap, // either su_ga or su_sga
3474 char_u *goodword,
3475 int badlenarg, // len of bad word replaced with "goodword"
3476 int score,
3477 int altscore,
3478 int had_bonus, // value for st_had_bonus
3479 slang_T *slang, // language for sound folding
3480 int maxsf) // su_maxscore applies to soundfold score,
3481 // su_sfmaxscore to the total score.
3482{
3483 int goodlen; // len of goodword changed
3484 int badlen; // len of bad word changed
3485 suggest_T *stp;
3486 suggest_T new_sug;
3487 int i;
3488 char_u *pgood, *pbad;
3489
3490 // Minimize "badlen" for consistency. Avoids that changing "the the" to
3491 // "thee the" is added next to changing the first "the" the "thee".
3492 pgood = goodword + STRLEN(goodword);
3493 pbad = su->su_badptr + badlenarg;
3494 for (;;)
3495 {
3496 goodlen = (int)(pgood - goodword);
3497 badlen = (int)(pbad - su->su_badptr);
3498 if (goodlen <= 0 || badlen <= 0)
3499 break;
3500 MB_PTR_BACK(goodword, pgood);
3501 MB_PTR_BACK(su->su_badptr, pbad);
3502 if (has_mbyte)
3503 {
3504 if (mb_ptr2char(pgood) != mb_ptr2char(pbad))
3505 break;
3506 }
3507 else if (*pgood != *pbad)
3508 break;
3509 }
3510
3511 if (badlen == 0 && goodlen == 0)
3512 // goodword doesn't change anything; may happen for "the the" changing
3513 // the first "the" to itself.
3514 return;
3515
3516 if (gap->ga_len == 0)
3517 i = -1;
3518 else
3519 {
3520 // Check if the word is already there. Also check the length that is
3521 // being replaced "thes," -> "these" is a different suggestion from
3522 // "thes" -> "these".
3523 stp = &SUG(*gap, 0);
3524 for (i = gap->ga_len; --i >= 0; ++stp)
3525 if (stp->st_wordlen == goodlen
3526 && stp->st_orglen == badlen
3527 && STRNCMP(stp->st_word, goodword, goodlen) == 0)
3528 {
3529 // Found it. Remember the word with the lowest score.
3530 if (stp->st_slang == NULL)
3531 stp->st_slang = slang;
3532
3533 new_sug.st_score = score;
3534 new_sug.st_altscore = altscore;
3535 new_sug.st_had_bonus = had_bonus;
3536
3537 if (stp->st_had_bonus != had_bonus)
3538 {
3539 // Only one of the two had the soundalike score computed.
3540 // Need to do that for the other one now, otherwise the
3541 // scores can't be compared. This happens because
3542 // suggest_try_change() doesn't compute the soundalike
3543 // word to keep it fast, while some special methods set
3544 // the soundalike score to zero.
3545 if (had_bonus)
3546 rescore_one(su, stp);
3547 else
3548 {
3549 new_sug.st_word = stp->st_word;
3550 new_sug.st_wordlen = stp->st_wordlen;
3551 new_sug.st_slang = stp->st_slang;
3552 new_sug.st_orglen = badlen;
3553 rescore_one(su, &new_sug);
3554 }
3555 }
3556
3557 if (stp->st_score > new_sug.st_score)
3558 {
3559 stp->st_score = new_sug.st_score;
3560 stp->st_altscore = new_sug.st_altscore;
3561 stp->st_had_bonus = new_sug.st_had_bonus;
3562 }
3563 break;
3564 }
3565 }
3566
3567 if (i < 0 && ga_grow(gap, 1) == OK)
3568 {
3569 // Add a suggestion.
3570 stp = &SUG(*gap, gap->ga_len);
3571 stp->st_word = vim_strnsave(goodword, goodlen);
3572 if (stp->st_word != NULL)
3573 {
3574 stp->st_wordlen = goodlen;
3575 stp->st_score = score;
3576 stp->st_altscore = altscore;
3577 stp->st_had_bonus = had_bonus;
3578 stp->st_orglen = badlen;
3579 stp->st_slang = slang;
3580 ++gap->ga_len;
3581
3582 // If we have too many suggestions now, sort the list and keep
3583 // the best suggestions.
3584 if (gap->ga_len > SUG_MAX_COUNT(su))
3585 {
3586 if (maxsf)
3587 su->su_sfmaxscore = cleanup_suggestions(gap,
3588 su->su_sfmaxscore, SUG_CLEAN_COUNT(su));
3589 else
3590 su->su_maxscore = cleanup_suggestions(gap,
3591 su->su_maxscore, SUG_CLEAN_COUNT(su));
3592 }
3593 }
3594 }
3595}
3596
3597/*
3598 * Suggestions may in fact be flagged as errors. Esp. for banned words and
3599 * for split words, such as "the the". Remove these from the list here.
3600 */
3601 static void
3602check_suggestions(
3603 suginfo_T *su,
3604 garray_T *gap) // either su_ga or su_sga
3605{
3606 suggest_T *stp;
3607 int i;
3608 char_u longword[MAXWLEN + 1];
3609 int len;
3610 hlf_T attr;
3611
Bram Moolenaar9c2b0662020-09-01 19:56:15 +02003612 if (gap->ga_len == 0)
3613 return;
Bram Moolenaar46a426c2019-09-27 12:41:56 +02003614 stp = &SUG(*gap, 0);
3615 for (i = gap->ga_len - 1; i >= 0; --i)
3616 {
3617 // Need to append what follows to check for "the the".
3618 vim_strncpy(longword, stp[i].st_word, MAXWLEN);
3619 len = stp[i].st_wordlen;
3620 vim_strncpy(longword + len, su->su_badptr + stp[i].st_orglen,
3621 MAXWLEN - len);
3622 attr = HLF_COUNT;
3623 (void)spell_check(curwin, longword, &attr, NULL, FALSE);
3624 if (attr != HLF_COUNT)
3625 {
3626 // Remove this entry.
3627 vim_free(stp[i].st_word);
3628 --gap->ga_len;
3629 if (i < gap->ga_len)
3630 mch_memmove(stp + i, stp + i + 1,
3631 sizeof(suggest_T) * (gap->ga_len - i));
3632 }
3633 }
3634}
3635
3636
3637/*
3638 * Add a word to be banned.
3639 */
3640 static void
3641add_banned(
3642 suginfo_T *su,
3643 char_u *word)
3644{
3645 char_u *s;
3646 hash_T hash;
3647 hashitem_T *hi;
3648
3649 hash = hash_hash(word);
3650 hi = hash_lookup(&su->su_banned, word, hash);
3651 if (HASHITEM_EMPTY(hi))
3652 {
3653 s = vim_strsave(word);
3654 if (s != NULL)
3655 hash_add_item(&su->su_banned, hi, s, hash);
3656 }
3657}
3658
3659/*
3660 * Recompute the score for all suggestions if sound-folding is possible. This
3661 * is slow, thus only done for the final results.
3662 */
3663 static void
3664rescore_suggestions(suginfo_T *su)
3665{
3666 int i;
3667
3668 if (su->su_sallang != NULL)
3669 for (i = 0; i < su->su_ga.ga_len; ++i)
3670 rescore_one(su, &SUG(su->su_ga, i));
3671}
3672
3673/*
3674 * Recompute the score for one suggestion if sound-folding is possible.
3675 */
3676 static void
3677rescore_one(suginfo_T *su, suggest_T *stp)
3678{
3679 slang_T *slang = stp->st_slang;
3680 char_u sal_badword[MAXWLEN];
3681 char_u *p;
3682
3683 // Only rescore suggestions that have no sal score yet and do have a
3684 // language.
3685 if (slang != NULL && slang->sl_sal.ga_len > 0 && !stp->st_had_bonus)
3686 {
3687 if (slang == su->su_sallang)
3688 p = su->su_sal_badword;
3689 else
3690 {
3691 spell_soundfold(slang, su->su_fbadword, TRUE, sal_badword);
3692 p = sal_badword;
3693 }
3694
3695 stp->st_altscore = stp_sal_score(stp, su, slang, p);
3696 if (stp->st_altscore == SCORE_MAXMAX)
3697 stp->st_altscore = SCORE_BIG;
3698 stp->st_score = RESCORE(stp->st_score, stp->st_altscore);
3699 stp->st_had_bonus = TRUE;
3700 }
3701}
3702
3703static int sug_compare(const void *s1, const void *s2);
3704
3705/*
3706 * Function given to qsort() to sort the suggestions on st_score.
3707 * First on "st_score", then "st_altscore" then alphabetically.
3708 */
3709 static int
3710sug_compare(const void *s1, const void *s2)
3711{
3712 suggest_T *p1 = (suggest_T *)s1;
3713 suggest_T *p2 = (suggest_T *)s2;
3714 int n = p1->st_score - p2->st_score;
3715
3716 if (n == 0)
3717 {
3718 n = p1->st_altscore - p2->st_altscore;
3719 if (n == 0)
3720 n = STRICMP(p1->st_word, p2->st_word);
3721 }
3722 return n;
3723}
3724
3725/*
3726 * Cleanup the suggestions:
3727 * - Sort on score.
3728 * - Remove words that won't be displayed.
3729 * Returns the maximum score in the list or "maxscore" unmodified.
3730 */
3731 static int
3732cleanup_suggestions(
3733 garray_T *gap,
3734 int maxscore,
3735 int keep) // nr of suggestions to keep
3736{
Bram Moolenaarbb65a562020-03-15 18:15:03 +01003737 if (gap->ga_len > 0)
Bram Moolenaar46a426c2019-09-27 12:41:56 +02003738 {
Bram Moolenaarbb65a562020-03-15 18:15:03 +01003739 // Sort the list.
3740 qsort(gap->ga_data, (size_t)gap->ga_len, sizeof(suggest_T),
3741 sug_compare);
3742
3743 // Truncate the list to the number of suggestions that will be
3744 // displayed.
3745 if (gap->ga_len > keep)
3746 {
Bram Moolenaar4ad739f2020-09-02 10:25:45 +02003747 int i;
3748 suggest_T *stp = &SUG(*gap, 0);
3749
Bram Moolenaarbb65a562020-03-15 18:15:03 +01003750 for (i = keep; i < gap->ga_len; ++i)
3751 vim_free(stp[i].st_word);
3752 gap->ga_len = keep;
3753 if (keep >= 1)
3754 return stp[keep - 1].st_score;
3755 }
Bram Moolenaar46a426c2019-09-27 12:41:56 +02003756 }
3757 return maxscore;
3758}
3759
3760/*
3761 * Compute a score for two sound-a-like words.
3762 * This permits up to two inserts/deletes/swaps/etc. to keep things fast.
3763 * Instead of a generic loop we write out the code. That keeps it fast by
3764 * avoiding checks that will not be possible.
3765 */
3766 static int
3767soundalike_score(
3768 char_u *goodstart, // sound-folded good word
3769 char_u *badstart) // sound-folded bad word
3770{
3771 char_u *goodsound = goodstart;
3772 char_u *badsound = badstart;
3773 int goodlen;
3774 int badlen;
3775 int n;
3776 char_u *pl, *ps;
3777 char_u *pl2, *ps2;
3778 int score = 0;
3779
3780 // Adding/inserting "*" at the start (word starts with vowel) shouldn't be
3781 // counted so much, vowels halfway the word aren't counted at all.
3782 if ((*badsound == '*' || *goodsound == '*') && *badsound != *goodsound)
3783 {
3784 if ((badsound[0] == NUL && goodsound[1] == NUL)
3785 || (goodsound[0] == NUL && badsound[1] == NUL))
3786 // changing word with vowel to word without a sound
3787 return SCORE_DEL;
3788 if (badsound[0] == NUL || goodsound[0] == NUL)
3789 // more than two changes
3790 return SCORE_MAXMAX;
3791
3792 if (badsound[1] == goodsound[1]
3793 || (badsound[1] != NUL
3794 && goodsound[1] != NUL
3795 && badsound[2] == goodsound[2]))
3796 {
3797 // handle like a substitute
3798 }
3799 else
3800 {
3801 score = 2 * SCORE_DEL / 3;
3802 if (*badsound == '*')
3803 ++badsound;
3804 else
3805 ++goodsound;
3806 }
3807 }
3808
3809 goodlen = (int)STRLEN(goodsound);
3810 badlen = (int)STRLEN(badsound);
3811
3812 // Return quickly if the lengths are too different to be fixed by two
3813 // changes.
3814 n = goodlen - badlen;
3815 if (n < -2 || n > 2)
3816 return SCORE_MAXMAX;
3817
3818 if (n > 0)
3819 {
3820 pl = goodsound; // goodsound is longest
3821 ps = badsound;
3822 }
3823 else
3824 {
3825 pl = badsound; // badsound is longest
3826 ps = goodsound;
3827 }
3828
3829 // Skip over the identical part.
3830 while (*pl == *ps && *pl != NUL)
3831 {
3832 ++pl;
3833 ++ps;
3834 }
3835
3836 switch (n)
3837 {
3838 case -2:
3839 case 2:
3840 // Must delete two characters from "pl".
3841 ++pl; // first delete
3842 while (*pl == *ps)
3843 {
3844 ++pl;
3845 ++ps;
3846 }
3847 // strings must be equal after second delete
3848 if (STRCMP(pl + 1, ps) == 0)
3849 return score + SCORE_DEL * 2;
3850
3851 // Failed to compare.
3852 break;
3853
3854 case -1:
3855 case 1:
3856 // Minimal one delete from "pl" required.
3857
3858 // 1: delete
3859 pl2 = pl + 1;
3860 ps2 = ps;
3861 while (*pl2 == *ps2)
3862 {
3863 if (*pl2 == NUL) // reached the end
3864 return score + SCORE_DEL;
3865 ++pl2;
3866 ++ps2;
3867 }
3868
3869 // 2: delete then swap, then rest must be equal
3870 if (pl2[0] == ps2[1] && pl2[1] == ps2[0]
3871 && STRCMP(pl2 + 2, ps2 + 2) == 0)
3872 return score + SCORE_DEL + SCORE_SWAP;
3873
3874 // 3: delete then substitute, then the rest must be equal
3875 if (STRCMP(pl2 + 1, ps2 + 1) == 0)
3876 return score + SCORE_DEL + SCORE_SUBST;
3877
3878 // 4: first swap then delete
3879 if (pl[0] == ps[1] && pl[1] == ps[0])
3880 {
3881 pl2 = pl + 2; // swap, skip two chars
3882 ps2 = ps + 2;
3883 while (*pl2 == *ps2)
3884 {
3885 ++pl2;
3886 ++ps2;
3887 }
3888 // delete a char and then strings must be equal
3889 if (STRCMP(pl2 + 1, ps2) == 0)
3890 return score + SCORE_SWAP + SCORE_DEL;
3891 }
3892
3893 // 5: first substitute then delete
3894 pl2 = pl + 1; // substitute, skip one char
3895 ps2 = ps + 1;
3896 while (*pl2 == *ps2)
3897 {
3898 ++pl2;
3899 ++ps2;
3900 }
3901 // delete a char and then strings must be equal
3902 if (STRCMP(pl2 + 1, ps2) == 0)
3903 return score + SCORE_SUBST + SCORE_DEL;
3904
3905 // Failed to compare.
3906 break;
3907
3908 case 0:
3909 // Lengths are equal, thus changes must result in same length: An
3910 // insert is only possible in combination with a delete.
3911 // 1: check if for identical strings
3912 if (*pl == NUL)
3913 return score;
3914
3915 // 2: swap
3916 if (pl[0] == ps[1] && pl[1] == ps[0])
3917 {
3918 pl2 = pl + 2; // swap, skip two chars
3919 ps2 = ps + 2;
3920 while (*pl2 == *ps2)
3921 {
3922 if (*pl2 == NUL) // reached the end
3923 return score + SCORE_SWAP;
3924 ++pl2;
3925 ++ps2;
3926 }
3927 // 3: swap and swap again
3928 if (pl2[0] == ps2[1] && pl2[1] == ps2[0]
3929 && STRCMP(pl2 + 2, ps2 + 2) == 0)
3930 return score + SCORE_SWAP + SCORE_SWAP;
3931
3932 // 4: swap and substitute
3933 if (STRCMP(pl2 + 1, ps2 + 1) == 0)
3934 return score + SCORE_SWAP + SCORE_SUBST;
3935 }
3936
3937 // 5: substitute
3938 pl2 = pl + 1;
3939 ps2 = ps + 1;
3940 while (*pl2 == *ps2)
3941 {
3942 if (*pl2 == NUL) // reached the end
3943 return score + SCORE_SUBST;
3944 ++pl2;
3945 ++ps2;
3946 }
3947
3948 // 6: substitute and swap
3949 if (pl2[0] == ps2[1] && pl2[1] == ps2[0]
3950 && STRCMP(pl2 + 2, ps2 + 2) == 0)
3951 return score + SCORE_SUBST + SCORE_SWAP;
3952
3953 // 7: substitute and substitute
3954 if (STRCMP(pl2 + 1, ps2 + 1) == 0)
3955 return score + SCORE_SUBST + SCORE_SUBST;
3956
3957 // 8: insert then delete
3958 pl2 = pl;
3959 ps2 = ps + 1;
3960 while (*pl2 == *ps2)
3961 {
3962 ++pl2;
3963 ++ps2;
3964 }
3965 if (STRCMP(pl2 + 1, ps2) == 0)
3966 return score + SCORE_INS + SCORE_DEL;
3967
3968 // 9: delete then insert
3969 pl2 = pl + 1;
3970 ps2 = ps;
3971 while (*pl2 == *ps2)
3972 {
3973 ++pl2;
3974 ++ps2;
3975 }
3976 if (STRCMP(pl2, ps2 + 1) == 0)
3977 return score + SCORE_INS + SCORE_DEL;
3978
3979 // Failed to compare.
3980 break;
3981 }
3982
3983 return SCORE_MAXMAX;
3984}
3985
3986/*
3987 * Compute the "edit distance" to turn "badword" into "goodword". The less
3988 * deletes/inserts/substitutes/swaps are required the lower the score.
3989 *
3990 * The algorithm is described by Du and Chang, 1992.
3991 * The implementation of the algorithm comes from Aspell editdist.cpp,
3992 * edit_distance(). It has been converted from C++ to C and modified to
3993 * support multi-byte characters.
3994 */
3995 static int
3996spell_edit_score(
3997 slang_T *slang,
3998 char_u *badword,
3999 char_u *goodword)
4000{
4001 int *cnt;
4002 int badlen, goodlen; // lengths including NUL
4003 int j, i;
4004 int t;
4005 int bc, gc;
4006 int pbc, pgc;
4007 char_u *p;
4008 int wbadword[MAXWLEN];
4009 int wgoodword[MAXWLEN];
4010
4011 if (has_mbyte)
4012 {
4013 // Get the characters from the multi-byte strings and put them in an
4014 // int array for easy access.
4015 for (p = badword, badlen = 0; *p != NUL; )
4016 wbadword[badlen++] = mb_cptr2char_adv(&p);
4017 wbadword[badlen++] = 0;
4018 for (p = goodword, goodlen = 0; *p != NUL; )
4019 wgoodword[goodlen++] = mb_cptr2char_adv(&p);
4020 wgoodword[goodlen++] = 0;
4021 }
4022 else
4023 {
4024 badlen = (int)STRLEN(badword) + 1;
4025 goodlen = (int)STRLEN(goodword) + 1;
4026 }
4027
4028 // We use "cnt" as an array: CNT(badword_idx, goodword_idx).
4029#define CNT(a, b) cnt[(a) + (b) * (badlen + 1)]
4030 cnt = ALLOC_MULT(int, (badlen + 1) * (goodlen + 1));
4031 if (cnt == NULL)
4032 return 0; // out of memory
4033
4034 CNT(0, 0) = 0;
4035 for (j = 1; j <= goodlen; ++j)
4036 CNT(0, j) = CNT(0, j - 1) + SCORE_INS;
4037
4038 for (i = 1; i <= badlen; ++i)
4039 {
4040 CNT(i, 0) = CNT(i - 1, 0) + SCORE_DEL;
4041 for (j = 1; j <= goodlen; ++j)
4042 {
4043 if (has_mbyte)
4044 {
4045 bc = wbadword[i - 1];
4046 gc = wgoodword[j - 1];
4047 }
4048 else
4049 {
4050 bc = badword[i - 1];
4051 gc = goodword[j - 1];
4052 }
4053 if (bc == gc)
4054 CNT(i, j) = CNT(i - 1, j - 1);
4055 else
4056 {
4057 // Use a better score when there is only a case difference.
4058 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc))
4059 CNT(i, j) = SCORE_ICASE + CNT(i - 1, j - 1);
4060 else
4061 {
4062 // For a similar character use SCORE_SIMILAR.
4063 if (slang != NULL
4064 && slang->sl_has_map
4065 && similar_chars(slang, gc, bc))
4066 CNT(i, j) = SCORE_SIMILAR + CNT(i - 1, j - 1);
4067 else
4068 CNT(i, j) = SCORE_SUBST + CNT(i - 1, j - 1);
4069 }
4070
4071 if (i > 1 && j > 1)
4072 {
4073 if (has_mbyte)
4074 {
4075 pbc = wbadword[i - 2];
4076 pgc = wgoodword[j - 2];
4077 }
4078 else
4079 {
4080 pbc = badword[i - 2];
4081 pgc = goodword[j - 2];
4082 }
4083 if (bc == pgc && pbc == gc)
4084 {
4085 t = SCORE_SWAP + CNT(i - 2, j - 2);
4086 if (t < CNT(i, j))
4087 CNT(i, j) = t;
4088 }
4089 }
4090 t = SCORE_DEL + CNT(i - 1, j);
4091 if (t < CNT(i, j))
4092 CNT(i, j) = t;
4093 t = SCORE_INS + CNT(i, j - 1);
4094 if (t < CNT(i, j))
4095 CNT(i, j) = t;
4096 }
4097 }
4098 }
4099
4100 i = CNT(badlen - 1, goodlen - 1);
4101 vim_free(cnt);
4102 return i;
4103}
4104
4105typedef struct
4106{
4107 int badi;
4108 int goodi;
4109 int score;
4110} limitscore_T;
4111
4112/*
4113 * Like spell_edit_score(), but with a limit on the score to make it faster.
4114 * May return SCORE_MAXMAX when the score is higher than "limit".
4115 *
4116 * This uses a stack for the edits still to be tried.
4117 * The idea comes from Aspell leditdist.cpp. Rewritten in C and added support
4118 * for multi-byte characters.
4119 */
4120 static int
4121spell_edit_score_limit(
4122 slang_T *slang,
4123 char_u *badword,
4124 char_u *goodword,
4125 int limit)
4126{
4127 limitscore_T stack[10]; // allow for over 3 * 2 edits
4128 int stackidx;
4129 int bi, gi;
4130 int bi2, gi2;
4131 int bc, gc;
4132 int score;
4133 int score_off;
4134 int minscore;
4135 int round;
4136
4137 // Multi-byte characters require a bit more work, use a different function
4138 // to avoid testing "has_mbyte" quite often.
4139 if (has_mbyte)
4140 return spell_edit_score_limit_w(slang, badword, goodword, limit);
4141
4142 // The idea is to go from start to end over the words. So long as
4143 // characters are equal just continue, this always gives the lowest score.
4144 // When there is a difference try several alternatives. Each alternative
4145 // increases "score" for the edit distance. Some of the alternatives are
4146 // pushed unto a stack and tried later, some are tried right away. At the
4147 // end of the word the score for one alternative is known. The lowest
4148 // possible score is stored in "minscore".
4149 stackidx = 0;
4150 bi = 0;
4151 gi = 0;
4152 score = 0;
4153 minscore = limit + 1;
4154
4155 for (;;)
4156 {
4157 // Skip over an equal part, score remains the same.
4158 for (;;)
4159 {
4160 bc = badword[bi];
4161 gc = goodword[gi];
4162 if (bc != gc) // stop at a char that's different
4163 break;
4164 if (bc == NUL) // both words end
4165 {
4166 if (score < minscore)
4167 minscore = score;
4168 goto pop; // do next alternative
4169 }
4170 ++bi;
4171 ++gi;
4172 }
4173
4174 if (gc == NUL) // goodword ends, delete badword chars
4175 {
4176 do
4177 {
4178 if ((score += SCORE_DEL) >= minscore)
4179 goto pop; // do next alternative
4180 } while (badword[++bi] != NUL);
4181 minscore = score;
4182 }
4183 else if (bc == NUL) // badword ends, insert badword chars
4184 {
4185 do
4186 {
4187 if ((score += SCORE_INS) >= minscore)
4188 goto pop; // do next alternative
4189 } while (goodword[++gi] != NUL);
4190 minscore = score;
4191 }
4192 else // both words continue
4193 {
4194 // If not close to the limit, perform a change. Only try changes
4195 // that may lead to a lower score than "minscore".
4196 // round 0: try deleting a char from badword
4197 // round 1: try inserting a char in badword
4198 for (round = 0; round <= 1; ++round)
4199 {
4200 score_off = score + (round == 0 ? SCORE_DEL : SCORE_INS);
4201 if (score_off < minscore)
4202 {
4203 if (score_off + SCORE_EDIT_MIN >= minscore)
4204 {
4205 // Near the limit, rest of the words must match. We
4206 // can check that right now, no need to push an item
4207 // onto the stack.
4208 bi2 = bi + 1 - round;
4209 gi2 = gi + round;
4210 while (goodword[gi2] == badword[bi2])
4211 {
4212 if (goodword[gi2] == NUL)
4213 {
4214 minscore = score_off;
4215 break;
4216 }
4217 ++bi2;
4218 ++gi2;
4219 }
4220 }
4221 else
4222 {
4223 // try deleting/inserting a character later
4224 stack[stackidx].badi = bi + 1 - round;
4225 stack[stackidx].goodi = gi + round;
4226 stack[stackidx].score = score_off;
4227 ++stackidx;
4228 }
4229 }
4230 }
4231
4232 if (score + SCORE_SWAP < minscore)
4233 {
4234 // If swapping two characters makes a match then the
4235 // substitution is more expensive, thus there is no need to
4236 // try both.
4237 if (gc == badword[bi + 1] && bc == goodword[gi + 1])
4238 {
4239 // Swap two characters, that is: skip them.
4240 gi += 2;
4241 bi += 2;
4242 score += SCORE_SWAP;
4243 continue;
4244 }
4245 }
4246
4247 // Substitute one character for another which is the same
4248 // thing as deleting a character from both goodword and badword.
4249 // Use a better score when there is only a case difference.
4250 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc))
4251 score += SCORE_ICASE;
4252 else
4253 {
4254 // For a similar character use SCORE_SIMILAR.
4255 if (slang != NULL
4256 && slang->sl_has_map
4257 && similar_chars(slang, gc, bc))
4258 score += SCORE_SIMILAR;
4259 else
4260 score += SCORE_SUBST;
4261 }
4262
4263 if (score < minscore)
4264 {
4265 // Do the substitution.
4266 ++gi;
4267 ++bi;
4268 continue;
4269 }
4270 }
4271pop:
4272 // Get here to try the next alternative, pop it from the stack.
4273 if (stackidx == 0) // stack is empty, finished
4274 break;
4275
4276 // pop an item from the stack
4277 --stackidx;
4278 gi = stack[stackidx].goodi;
4279 bi = stack[stackidx].badi;
4280 score = stack[stackidx].score;
4281 }
4282
4283 // When the score goes over "limit" it may actually be much higher.
4284 // Return a very large number to avoid going below the limit when giving a
4285 // bonus.
4286 if (minscore > limit)
4287 return SCORE_MAXMAX;
4288 return minscore;
4289}
4290
4291/*
4292 * Multi-byte version of spell_edit_score_limit().
4293 * Keep it in sync with the above!
4294 */
4295 static int
4296spell_edit_score_limit_w(
4297 slang_T *slang,
4298 char_u *badword,
4299 char_u *goodword,
4300 int limit)
4301{
4302 limitscore_T stack[10]; // allow for over 3 * 2 edits
4303 int stackidx;
4304 int bi, gi;
4305 int bi2, gi2;
4306 int bc, gc;
4307 int score;
4308 int score_off;
4309 int minscore;
4310 int round;
4311 char_u *p;
4312 int wbadword[MAXWLEN];
4313 int wgoodword[MAXWLEN];
4314
4315 // Get the characters from the multi-byte strings and put them in an
4316 // int array for easy access.
4317 bi = 0;
4318 for (p = badword; *p != NUL; )
4319 wbadword[bi++] = mb_cptr2char_adv(&p);
4320 wbadword[bi++] = 0;
4321 gi = 0;
4322 for (p = goodword; *p != NUL; )
4323 wgoodword[gi++] = mb_cptr2char_adv(&p);
4324 wgoodword[gi++] = 0;
4325
4326 // The idea is to go from start to end over the words. So long as
4327 // characters are equal just continue, this always gives the lowest score.
4328 // When there is a difference try several alternatives. Each alternative
4329 // increases "score" for the edit distance. Some of the alternatives are
4330 // pushed unto a stack and tried later, some are tried right away. At the
4331 // end of the word the score for one alternative is known. The lowest
4332 // possible score is stored in "minscore".
4333 stackidx = 0;
4334 bi = 0;
4335 gi = 0;
4336 score = 0;
4337 minscore = limit + 1;
4338
4339 for (;;)
4340 {
4341 // Skip over an equal part, score remains the same.
4342 for (;;)
4343 {
4344 bc = wbadword[bi];
4345 gc = wgoodword[gi];
4346
4347 if (bc != gc) // stop at a char that's different
4348 break;
4349 if (bc == NUL) // both words end
4350 {
4351 if (score < minscore)
4352 minscore = score;
4353 goto pop; // do next alternative
4354 }
4355 ++bi;
4356 ++gi;
4357 }
4358
4359 if (gc == NUL) // goodword ends, delete badword chars
4360 {
4361 do
4362 {
4363 if ((score += SCORE_DEL) >= minscore)
4364 goto pop; // do next alternative
4365 } while (wbadword[++bi] != NUL);
4366 minscore = score;
4367 }
4368 else if (bc == NUL) // badword ends, insert badword chars
4369 {
4370 do
4371 {
4372 if ((score += SCORE_INS) >= minscore)
4373 goto pop; // do next alternative
4374 } while (wgoodword[++gi] != NUL);
4375 minscore = score;
4376 }
4377 else // both words continue
4378 {
4379 // If not close to the limit, perform a change. Only try changes
4380 // that may lead to a lower score than "minscore".
4381 // round 0: try deleting a char from badword
4382 // round 1: try inserting a char in badword
4383 for (round = 0; round <= 1; ++round)
4384 {
4385 score_off = score + (round == 0 ? SCORE_DEL : SCORE_INS);
4386 if (score_off < minscore)
4387 {
4388 if (score_off + SCORE_EDIT_MIN >= minscore)
4389 {
4390 // Near the limit, rest of the words must match. We
4391 // can check that right now, no need to push an item
4392 // onto the stack.
4393 bi2 = bi + 1 - round;
4394 gi2 = gi + round;
4395 while (wgoodword[gi2] == wbadword[bi2])
4396 {
4397 if (wgoodword[gi2] == NUL)
4398 {
4399 minscore = score_off;
4400 break;
4401 }
4402 ++bi2;
4403 ++gi2;
4404 }
4405 }
4406 else
4407 {
4408 // try deleting a character from badword later
4409 stack[stackidx].badi = bi + 1 - round;
4410 stack[stackidx].goodi = gi + round;
4411 stack[stackidx].score = score_off;
4412 ++stackidx;
4413 }
4414 }
4415 }
4416
4417 if (score + SCORE_SWAP < minscore)
4418 {
4419 // If swapping two characters makes a match then the
4420 // substitution is more expensive, thus there is no need to
4421 // try both.
4422 if (gc == wbadword[bi + 1] && bc == wgoodword[gi + 1])
4423 {
4424 // Swap two characters, that is: skip them.
4425 gi += 2;
4426 bi += 2;
4427 score += SCORE_SWAP;
4428 continue;
4429 }
4430 }
4431
4432 // Substitute one character for another which is the same
4433 // thing as deleting a character from both goodword and badword.
4434 // Use a better score when there is only a case difference.
4435 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc))
4436 score += SCORE_ICASE;
4437 else
4438 {
4439 // For a similar character use SCORE_SIMILAR.
4440 if (slang != NULL
4441 && slang->sl_has_map
4442 && similar_chars(slang, gc, bc))
4443 score += SCORE_SIMILAR;
4444 else
4445 score += SCORE_SUBST;
4446 }
4447
4448 if (score < minscore)
4449 {
4450 // Do the substitution.
4451 ++gi;
4452 ++bi;
4453 continue;
4454 }
4455 }
4456pop:
4457 // Get here to try the next alternative, pop it from the stack.
4458 if (stackidx == 0) // stack is empty, finished
4459 break;
4460
4461 // pop an item from the stack
4462 --stackidx;
4463 gi = stack[stackidx].goodi;
4464 bi = stack[stackidx].badi;
4465 score = stack[stackidx].score;
4466 }
4467
4468 // When the score goes over "limit" it may actually be much higher.
4469 // Return a very large number to avoid going below the limit when giving a
4470 // bonus.
4471 if (minscore > limit)
4472 return SCORE_MAXMAX;
4473 return minscore;
4474}
4475#endif // FEAT_SPELL