patch 8.0.0645: no error for illegal back reference in NFA engine

Problem:    The new regexp engine does not give an error for using a back
            reference where it is not allowed. (Dominique Pelle)
Solution:   Check the back reference like the old engine. (closes #1774)
diff --git a/src/testdir/test_hlsearch.vim b/src/testdir/test_hlsearch.vim
index b934d38..e9790bb 100644
--- a/src/testdir/test_hlsearch.vim
+++ b/src/testdir/test_hlsearch.vim
@@ -38,11 +38,11 @@
     return
   endif
 
-  " This pattern takes forever to match, it should timeout.
+  " This pattern takes a long time to match, it should timeout.
   help
   let start = reltime()
   set hlsearch nolazyredraw redrawtime=101
-  let @/ = '\%#=2\v(a|\1)*'
+  let @/ = '\%#=1a*.*X\@<=b*'
   redraw
   let elapsed = reltimefloat(reltime(start))
   call assert_true(elapsed > 0.1)