diff --git a/src/main/java/org/sablecc/sablecc/lexer/Lexer.java b/src/main/java/org/sablecc/sablecc/lexer/Lexer.java index 438b84dc1eeb712f6086a4e75e3eaaa9f9a2588f..c38142abd0ab59a90dc6849b637833b758bb01b7 100644 --- a/src/main/java/org/sablecc/sablecc/lexer/Lexer.java +++ b/src/main/java/org/sablecc/sablecc/lexer/Lexer.java @@ -390,6 +390,19 @@ public class Lexer } /** + * <p> + * Push the given token's text back onto the input. + * Note that the lexer state is <i>not</i> restored, + * so a following {@link #next()}/{@link #peek()} call may result in a different token. + * </p> + * <p> + * <b>Note:</b> + * If the token text contains newlines, + * the caller must ensure that CR+LF pairs are unread in their entirety. + * If only one half of a CR+LF pair is unread, + * the line numbers will be incorrect when it is lexed again. + * </p> + * * @param tok the token to push back onto the input * @throws IOException when thrown by {@link PushbackReader#unread(int)} */ @@ -397,10 +410,6 @@ public class Lexer { String tokenText = tok.getText(); int length = tokenText.length(); - if(this.cr || (length > 0 && tokenText.charAt(0) == '\n')) - { - throw new IOException("Cannot unread a token containing a partial newline"); - } for(int i = length - 1; i >= 0; i--) { diff --git a/src/main/resources/org/sablecc/sablecc/lexer.txt b/src/main/resources/org/sablecc/sablecc/lexer.txt index 1577bea9cee383c6a0c78aba74cd632408dd48af..c877b18803adbde214f1aeb8ac1584b94194fd03 100644 --- a/src/main/resources/org/sablecc/sablecc/lexer.txt +++ b/src/main/resources/org/sablecc/sablecc/lexer.txt @@ -284,6 +284,19 @@ Macro:LexerBody } /** + * <p> + * Push the given token's text back onto the input. + * Note that the lexer state is <i>not</i> restored, + * so a following {@link #next()}/{@link #peek()} call may result in a different token. + * </p> + * <p> + * <b>Note:</b> + * If the token text contains newlines, + * the caller must ensure that CR+LF pairs are unread in their entirety. + * If only one half of a CR+LF pair is unread, + * the line numbers will be incorrect when it is lexed again. + * </p> + * * @param tok the token to push back onto the input * @throws IOException when thrown by {@link PushbackReader#unread(int)} */ @@ -291,10 +304,6 @@ Macro:LexerBody { String tokenText = tok.getText(); int length = tokenText.length(); - if(this.cr || (length > 0 && tokenText.charAt(0) == '\n')) - { - throw new IOException("Cannot unread a token containing a partial newline"); - } for(int i = length - 1; i >= 0; i--) {