Use syntax-propertize-function, not font-lock-syntactic-keywords

font-lock-syntactic-keywords have been obsolete since Emacs 24.1, the
earliest version supported by rust-mode.  Instead, modes are
encouraged to use syntax-propertize-function.

syntax-propertize-function provides a generally better experience.
Syntax propertization is not tied to font lock,so angle bracket
matching will still work when font-lock is disabled.  There's no
longer a need to call font-lock-fontify-buffer or font-lock-ensure
(you can see this in the tests).  The resulting code is also shorter.

I removed a few tests:

* font-lock-raw-string-constant was written assuming font-lock based
  syntax propertization, but that's no longer the case.
* font-lock-extend-region-in-string and
  rust-test-revert-hook-preserves-point both called functions that no
  longer exist.

There's a fix for a hidden bug in rust-is-in-expression-context.  This
previously could signal in some situations, but the signal was hidden
by font-lock.  It was visible now when running tests, hence the new
call to condition-case.

I suspect this might fix #192, but I haven't tried it.

1 files changed, 107 insertions, 222 deletions

diff --git a/rust-mode.el b/rust-mode.el
index dfe3e9a..47c6661 100644
--- a/rust-mode.el
+++ b/rust-mode.el
@@ -35,67 +35,6 @@
 (defconst rust-re-unsafe "unsafe")
 (defconst rust-re-extern "extern")
 
-(defconst rust-re-non-standard-string
-  (rx
-   (or
-    ;; Raw string: if it matches, it ends up with the starting character
-    ;; of the string as group 1, any ending backslashes as group 4, and
-    ;; the ending character as either group 5 or group 6.
-    (seq
-     ;; The "r" starts the raw string.  Capture it as group 1 to mark it as such syntactically:
-     (group "r")
-
-     ;; Then either:
-     (or
-      ;; a sequence at least one "#" (followed by quote).  Capture all
-      ;; but the last "#" as group 2 for this case.
-      (seq (group (* "#")) "#\"")
-
-      ;; ...or a quote without any "#".  Capture it as group 3. This is
-      ;; used later to match the opposite quote only if this capture
-      ;; occurred
-      (group "\""))
-
-     ;; The contents of the string:
-     (*? anything)
-
-     ;; If there are any backslashes at the end of the string, capture
-     ;; them as group 4 so we can suppress the normal escape syntax
-     ;; parsing:
-     (group (* "\\"))
-
-     ;; Then the end of the string--the backreferences ensure that we
-     ;; only match the kind of ending that corresponds to the beginning
-     ;; we had:
-     (or
-      ;; There were "#"s - capture the last one as group 5 to mark it as
-      ;; the end of the string:
-      (seq "\"" (backref 2) (group "#"))
-
-      ;; No "#"s - capture the ending quote (using a backref to group 3,
-      ;; so that we can't match a quote if we had "#"s) as group 6
-      (group (backref 3))
-
-      ;; If the raw string wasn't actually closed, go all the way to the end
-      string-end))
-
-    ;; Character literal: match the beginning ' of a character literal
-    ;; as group 7, and the ending one as group 8
-    (seq
-     (group "'")
-     (or
-      (seq
-       "\\"
-       (or
-        (: "u{" (** 1 6 xdigit) "}")
-        (: "x" (= 2 xdigit))
-        (any "'nrt0\"\\")))
-      (not (any "'\\"))
-      )
-     (group "'"))
-    )
-   ))
-
 (defun rust-looking-back-str (str)
   "Like `looking-back' but for fixed strings rather than regexps (so that it's not so slow)"
   (let ((len (length str)))
@@ -133,8 +72,8 @@
     (modify-syntax-entry ?\" "\"" table)
     (modify-syntax-entry ?\\ "\\" table)
 
-    ;; Angle brackets.  We suppress this with syntactic fontification when
-    ;; needed
+    ;; Angle brackets.  We suppress this with syntactic propertization
+    ;; when needed
     (modify-syntax-entry ?< "(>" table)
     (modify-syntax-entry ?> ")<" table)
 
@@ -643,94 +582,6 @@ the desired identifiers), but does not match type annotations \"foo::<\"."
              ("use" . font-lock-constant-face)
              ("fn" . font-lock-function-name-face)))))
 
-(defvar font-lock-beg)
-(defvar font-lock-end)
-
-(defun rust-font-lock-extend-region ()
-  "Extend the region given by `font-lock-beg' and `font-lock-end'
-  to include the beginning of a string or comment if it includes
-  part of it.  Adjusts to include the r[#] of a raw string as
-  well."
-
-  (save-excursion
-    (let ((orig-beg font-lock-beg)
-          (orig-end font-lock-end))
-
-      (let*
-          ;; It's safe to call `syntax-ppss' here on positions that are
-          ;; already syntactically fontified
-          ((beg-ppss (syntax-ppss font-lock-beg))
-           (beg-in-cmnt (and beg-ppss (nth 4 beg-ppss) (nth 8 beg-ppss)))
-           (beg-in-str (and beg-ppss (nth 3 beg-ppss) (nth 8 beg-ppss))))
-
-        (when (and beg-in-str (>= font-lock-beg beg-in-str))
-          (setq font-lock-beg (nth 8 beg-ppss))
-          (while (equal ?# (char-before font-lock-beg))
-            (setq font-lock-beg (1- font-lock-beg)))
-          (when (equal ?r (char-before font-lock-beg))
-            (setq font-lock-beg (1- font-lock-beg))))
-
-        (when (and beg-in-cmnt (> font-lock-beg beg-in-cmnt))
-          (setq font-lock-beg beg-in-cmnt)))
-
-      ;; We need to make sure that if the region ends inside a raw string, we
-      ;; extend it out past the end of it.  But we can't use `syntax-ppss' to
-      ;; detect that, becaue that depends on font-lock already being done, and we
-      ;; are trying to figure out how much to font-lock before that.  So we use
-      ;; the regexp directly.
-      (save-match-data
-        (goto-char font-lock-beg)
-        (while (and (< (point) font-lock-end)
-                    (re-search-forward rust-re-non-standard-string (buffer-end 1) t)
-                    (<= (match-beginning 0) font-lock-end))
-          (setq font-lock-end (max font-lock-end (match-end 0)))
-          (goto-char (1+ (match-beginning 0)))))
-
-      (or (/= font-lock-beg orig-beg)
-          (/= font-lock-end orig-end))
-      )))
-
-(defun rust-conditional-re-search-forward (regexp bound condition)
-  ;; Search forward for regexp (with bound).  If found, call condition and return the found
-  ;; match only if it returns true.
-  (let* (found
-         found-ret-list
-         (ret-list (save-excursion
-                     (while (and (not found) (re-search-forward regexp bound t))
-                       (setq
-                        found-ret-list (list (point) (match-data))
-                        found (save-match-data (save-excursion (ignore-errors (funcall condition)))))
-                       ;; If the condition filters out a match, need to search
-                       ;; again just after its beginning.  This will allow
-                       ;; cases such as:
-                       ;;    "bar" r"foo"
-                       ;; where the filtered out search (r" r") should not
-                       ;; prevent finding another one that begins in the middle
-                       ;; of it (r"foo")
-                       (when (not found)
-                         (goto-char (1+ (match-beginning 0))))
-                       )
-                     (when found found-ret-list))))
-    (when ret-list
-      (goto-char (nth 0 ret-list))
-      (set-match-data (nth 1 ret-list))
-      (nth 0 ret-list))))
-
-(defun rust-look-for-non-standard-string (bound)
-  ;; Find a raw string or character literal, but only if it's not in the middle
-  ;; of another string or a comment.
-
-  (rust-conditional-re-search-forward
-   rust-re-non-standard-string
-   bound
-   (lambda ()
-     (let ((pstate (syntax-ppss (match-beginning 0))))
-       (not
-        (or
-         (nth 4 pstate) ;; Skip if in a comment
-         (and (nth 3 pstate) (wholenump (nth 8 pstate)) (< (nth 8 pstate) (match-beginning 0))) ;; Skip if in a string that isn't starting here
-         ))))))
-
 (defun rust-syntax-class-before-point ()
   (when (> (point) 1)
     (syntax-class (syntax-after (1- (point))))))
@@ -960,8 +811,11 @@ the desired identifiers), but does not match type annotations \"foo::<\"."
        ((or
          (equal 4 (rust-syntax-class-before-point))
          (rust-looking-back-str ","))
-        (backward-up-list)
-        (rust-is-in-expression-context 'open-brace))
+	(condition-case nil
+	    (progn
+	      (backward-up-list)
+	      (rust-is-in-expression-context 'open-brace))
+	  (scan-error nil)))
 
        ;; A => introduces an expression
        ((rust-looking-back-str "=>") t)
@@ -1037,56 +891,42 @@ the desired identifiers), but does not match type annotations \"foo::<\"."
        (rust-is-lt-char-operator)))
    (funcall (default-value 'electric-pair-inhibit-predicate) char)))
 
-(defun rust-look-for-non-angle-bracket-lt-gt (bound)
-  "Find an angle bracket (\"<\" or \">\") that should be part of
-  a matched pair Relies on the fact that when it finds a < or >,
-  we have already decided which previous ones are angle brackets
-  and which ones are not.  So this only really works as a
-  font-lock-syntactic-keywords matcher--it won't work at
-  arbitrary positions without the earlier parts of the buffer
-  having already been covered."
-
-  (rust-conditional-re-search-forward
-   "[<>]" bound
-   (lambda ()
-     (goto-char (match-beginning 0))
-     (cond
-      ;; If matching is turned off suppress all of them
-      ((not rust-match-angle-brackets) t)
-
-      ;; We don't take < or > in strings or comments to be angle brackets
-      ((rust-in-str-or-cmnt) t)
-
-      ;; Inside a macro we don't really know the syntax.  Any < or > may be an
-      ;; angle bracket or it may not.  But we know that the other braces have
-      ;; to balance regardless of the < and >, so if we don't treat any < or >
-      ;; as angle brackets it won't mess up any paren balancing.
-      ((rust-in-macro) t)
-      
-      ((looking-at "<")
-       (rust-is-lt-char-operator))
+(defun rust-ordinary-lt-gt-p ()
+  "Test whether the `<' or `>' at point is an ordinary operator of some kind.
 
-      ((looking-at ">")
-       (cond
-        ;; Don't treat the > in -> or => as an angle bracket
-        ((member (char-before (point)) '(?- ?=)) t)
-
-        ;; If we are at top level and not in any list, it can't be a closing
-        ;; angle bracket
-        ((>= 0 (rust-paren-level)) t)
-
-        ;; Otherwise, treat the > as a closing angle bracket if it would
-        ;; match an opening one
-        ((save-excursion
-           (backward-up-list)
-           (not (looking-at "<"))))))))))
-
-(defvar rust-mode-font-lock-syntactic-keywords
-  (append
-   ;; Handle raw strings and character literals:
-   `((rust-look-for-non-standard-string (1 "|" nil t) (4 "_" nil t) (5 "|" nil t) (6 "|" nil t) (7 "\"" nil t) (8 "\"" nil t)))
-   ;; Find where < and > characters represent operators rather than angle brackets:
-   '((rust-look-for-non-angle-bracket-lt-gt (0 "." t)))))
+This returns t if the `<' or `>' is an ordinary operator (like
+less-than) or part of one (like `->'); and nil if the character
+should be considered a paired angle bracket."
+  (cond
+   ;; If matching is turned off suppress all of them
+   ((not rust-match-angle-brackets) t)
+
+   ;; We don't take < or > in strings or comments to be angle brackets
+   ((rust-in-str-or-cmnt) t)
+
+   ;; Inside a macro we don't really know the syntax.  Any < or > may be an
+   ;; angle bracket or it may not.  But we know that the other braces have
+   ;; to balance regardless of the < and >, so if we don't treat any < or >
+   ;; as angle brackets it won't mess up any paren balancing.
+   ((rust-in-macro) t)
+   
+   ((looking-at "<")
+    (rust-is-lt-char-operator))
+
+   ((looking-at ">")
+    (cond
+     ;; Don't treat the > in -> or => as an angle bracket
+     ((member (char-before (point)) '(?- ?=)) t)
+
+     ;; If we are at top level and not in any list, it can't be a closing
+     ;; angle bracket
+     ((>= 0 (rust-paren-level)) t)
+
+     ;; Otherwise, treat the > as a closing angle bracket if it would
+     ;; match an opening one
+     ((save-excursion
+	(backward-up-list)
+	(not (looking-at "<"))))))))
 
 (defun rust-mode-syntactic-face-function (state)
   "Syntactic face function to distinguish doc comments from other comments."
@@ -1098,6 +938,68 @@ the desired identifiers), but does not match type annotations \"foo::<\"."
         'font-lock-comment-face
     ))))
 
+(eval-and-compile
+  (defconst rust--char-literal-rx
+    (rx (seq
+	 (group "'")
+	 (or
+	  (seq
+	   "\\"
+	   (or
+	    (: "u{" (** 1 6 xdigit) "}")
+	    (: "x" (= 2 xdigit))
+	    (any "'nrt0\"\\")))
+	  (not (any "'\\"))
+	  )
+	 (group "'")))
+    "A regular expression matching a character literal."))
+
+(defun rust--syntax-propertize-raw-string (end)
+  "A helper for rust-syntax-propertize.
+
+If point is already in a raw string, this will apply the
+appropriate string syntax to the character up to the end of the
+raw string, or to `end', whichever comes first."
+  (let ((str-start (nth 8 (syntax-ppss))))
+    (when str-start
+      (when (save-excursion
+	      (goto-char str-start)
+	      (looking-at "r\\(#*\\)\\(\"\\)"))
+	;; In a raw string, so try to find the end.
+	(let ((hashes (match-string 1)))
+	  ;; Match \ characters at the end of the string to suppress
+	  ;; their normal character-quote syntax.
+	  (when (re-search-forward (concat "\\(\\\\*\\)\\(\"" hashes "\\)") end t)
+	    (put-text-property (match-beginning 1) (match-end 1)
+			       'syntax-table (string-to-syntax "_"))
+	    (put-text-property (1- (match-end 2)) (match-end 2)
+			       'syntax-table (string-to-syntax "|"))
+	    (goto-char (match-end 0))))))))
+
+(defun rust-syntax-propertize (start end)
+  "A `syntax-propertize-function' for `rust-mode'."
+  (goto-char start)
+  (rust--syntax-propertize-raw-string end)
+  (funcall
+   (syntax-propertize-rules
+    ;; Character literals.
+    (rust--char-literal-rx (1 "\"") (2 "\""))
+    ;; Raw strings.
+    ("\\(r\\)#*\""
+     (1 (prog1 "|"
+	  (goto-char (match-end 0))
+	  (rust--syntax-propertize-raw-string end))))
+    ("[<>]"
+     (0 (ignore
+	 (when (save-match-data
+		 (save-excursion
+		   (goto-char (match-beginning 0))
+		   (rust-ordinary-lt-gt-p)))
+	   (put-text-property (match-beginning 0) (match-end 0)
+			      'syntax-table (string-to-syntax "."))
+	   (goto-char (match-end 0)))))))
+   (point) end))
+
 (defun rust-fill-prefix-for-comment-start (line-start)
   "Determine what to use for `fill-prefix' based on what is at the beginning of a line."
   (let ((result
@@ -1410,11 +1312,6 @@ This is written mainly to be used as `end-of-defun-function' for Rust."
         (set-window-start window start)
         (set-window-point window pos))))
 
-  ;; Issue #127: Running this on a buffer acts like a revert, and could cause
-  ;; the fontification to get out of sync.  Call the same hook to ensure it is
-  ;; restored.
-  (rust--after-revert-hook)
-
   (message "Formatted buffer with rustfmt."))
 
 (defun rust-enable-format-on-save ()
@@ -1441,14 +1338,15 @@ This is written mainly to be used as `end-of-defun-function' for Rust."
   :group 'rust-mode
   :syntax-table rust-mode-syntax-table
 
+  ;; Syntax.
+  (setq-local syntax-propertize-function #'rust-syntax-propertize)
+
   ;; Indentation
   (setq-local indent-line-function 'rust-mode-indent-line)
 
   ;; Fonts
-  (add-to-list 'font-lock-extend-region-functions 'rust-font-lock-extend-region)
   (setq-local font-lock-defaults '(rust-mode-font-lock-keywords
                                    nil nil nil nil
-                                   (font-lock-syntactic-keywords . rust-mode-font-lock-syntactic-keywords)
                                    (font-lock-syntactic-face-function . rust-mode-syntactic-face-function)
                                    ))
 
@@ -1480,7 +1378,6 @@ This is written mainly to be used as `end-of-defun-function' for Rust."
   (setq-local end-of-defun-function 'rust-end-of-defun)
   (setq-local parse-sexp-lookup-properties t)
   (setq-local electric-pair-inhibit-predicate 'rust-electric-pair-inhibit-predicate-wrap)
-  (add-hook 'after-revert-hook 'rust--after-revert-hook nil t)
   (add-hook 'before-save-hook 'rust--before-save-hook nil t))
 
 ;;;###autoload
@@ -1492,18 +1389,6 @@ This is written mainly to be used as `end-of-defun-function' for Rust."
   (require 'rust-mode)
   (rust-mode))
 
-;; Issue #104: When reverting the buffer, make sure all fontification is redone
-;; so that we don't end up missing a non-angle-bracket '<' or '>' character.
-(defun rust--after-revert-hook ()
-  ;; In Emacs 25 and later, the preferred method to force fontification is
-  ;; to use `font-lock-ensure', which doesn't exist in Emacs 24 and earlier.
-  ;; If it's not available, fall back to calling `font-lock-fontify-region'
-  ;; on the whole buffer.
-  (save-excursion
-    (if (fboundp 'font-lock-ensure)
-        (font-lock-ensure)
-      (font-lock-fontify-region (point-min) (point-max)))))
-
 (defun rust--before-save-hook ()
   (when rust-format-on-save (rust-format-buffer)))