From c81c2a51ec8509f3ff497d55432e64120e6ddc13 Mon Sep 17 00:00:00 2001
From: dgelessus <dgelessus@users.noreply.github.com>
Date: Fri, 7 Jul 2023 11:11:31 +0200
Subject: [PATCH] Deprecate old source position APIs that are bad for memory
usage
---
.../java/de/hhu/stups/sablecc/patch/IParser.java | 5 +++++
.../stups/sablecc/patch/ITokenListContainer.java | 10 ++++++++++
.../de/hhu/stups/sablecc/patch/SourcePositions.java | 5 +++++
.../de/hhu/stups/sablecc/patch/SourcecodeRange.java | 1 +
src/main/java/org/sablecc/sablecc/lexer/Lexer.java | 13 +++++++++++--
.../java/org/sablecc/sablecc/parser/Parser.java | 9 +++++++--
src/main/resources/org/sablecc/sablecc/lexer.txt | 13 +++++++++++--
src/main/resources/org/sablecc/sablecc/parser.txt | 9 +++++++--
8 files changed, 57 insertions(+), 8 deletions(-)
diff --git a/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/IParser.java b/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/IParser.java
index 1dcb958..ba7e1bd 100644
--- a/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/IParser.java
+++ b/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/IParser.java
@@ -10,5 +10,10 @@ import java.util.Map;
public interface IParser {
+ /**
+ * @deprecated All generated token classes store their own position info.
+ * Use the {@link PositionedNode} or {@link IToken} APIs to access the positions stored in the tokens.
+ */
+ @Deprecated
public Map<PositionedNode, SourcecodeRange> getMapping();
}
diff --git a/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/ITokenListContainer.java b/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/ITokenListContainer.java
index eb5eb88..eb07fe5 100644
--- a/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/ITokenListContainer.java
+++ b/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/ITokenListContainer.java
@@ -8,6 +8,16 @@ package de.hhu.stups.sablecc.patch;
import java.util.List;
+/**
+ * @deprecated In the future, the generated lexers will not store all tokens anymore, because this leads to significant memory usage for large inputs.
+ * If you really need this list, please create it yourself, using a custom lexer subclass if necessary.
+ */
+@Deprecated
public interface ITokenListContainer {
+ /**
+ * @deprecated In the future, the generated lexers will not store all tokens anymore, because this leads to significant memory usage for large inputs.
+ * If you really need this list, please create it yourself, using a custom lexer subclass if necessary.
+ */
+ @Deprecated
public List<IToken> getTokenList();
}
diff --git a/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/SourcePositions.java b/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/SourcePositions.java
index 0c30041..7dc66fd 100644
--- a/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/SourcePositions.java
+++ b/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/SourcePositions.java
@@ -10,6 +10,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+/**
+ * @deprecated All generated token classes store their own position info.
+ * Use the {@link PositionedNode} or {@link IToken} APIs to access the positions stored in the tokens.
+ */
+@Deprecated
public class SourcePositions {
private final List<IToken> tokenList;
private final Map<PositionedNode, SourcecodeRange> positions;
diff --git a/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/SourcecodeRange.java b/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/SourcecodeRange.java
index 7f25cd1..f447a79 100644
--- a/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/SourcecodeRange.java
+++ b/sablecc-runtime/src/main/java/de/hhu/stups/sablecc/patch/SourcecodeRange.java
@@ -6,6 +6,7 @@
package de.hhu.stups.sablecc.patch;
+@Deprecated
public class SourcecodeRange {
private final int beginIndex;
private final int endIndex;
diff --git a/src/main/java/org/sablecc/sablecc/lexer/Lexer.java b/src/main/java/org/sablecc/sablecc/lexer/Lexer.java
index 9a4ac88..88b92d8 100644
--- a/src/main/java/org/sablecc/sablecc/lexer/Lexer.java
+++ b/src/main/java/org/sablecc/sablecc/lexer/Lexer.java
@@ -17,7 +17,7 @@ import de.hhu.stups.sablecc.patch.ITokenListContainer;
import org.sablecc.sablecc.node.*;
-@SuppressWarnings({"unused"})
+@SuppressWarnings({"deprecation", "unused"}) // ITokenListContainer is deprecated, but the generated lexer still implements it for compatibility
public class Lexer implements ITokenListContainer
{
protected Token token;
@@ -37,6 +37,11 @@ public class Lexer implements ITokenListContainer
return nextList;
}
+ /**
+ * @deprecated In the future, the generated lexers will not store all tokens anymore, because this leads to significant memory usage for large inputs.
+ * If you really need this list, please create it yourself, using a custom lexer subclass if necessary.
+ */
+ @Deprecated
@Override
public List<IToken> getTokenList() {
return tokenList;
@@ -46,7 +51,11 @@ public class Lexer implements ITokenListContainer
token = t;
}
-
+ /**
+ * @deprecated In the future, the generated lexers will not store all tokens anymore, because this leads to significant memory usage for large inputs.
+ * If you really need this list, please create it yourself, using a custom lexer subclass if necessary.
+ */
+ @Deprecated
public void setTokenList(final List<IToken> list) {
tokenList = list;
}
diff --git a/src/main/java/org/sablecc/sablecc/parser/Parser.java b/src/main/java/org/sablecc/sablecc/parser/Parser.java
index a5f0174..28bbe35 100644
--- a/src/main/java/org/sablecc/sablecc/parser/Parser.java
+++ b/src/main/java/org/sablecc/sablecc/parser/Parser.java
@@ -22,7 +22,7 @@ import org.sablecc.sablecc.lexer.Lexer;
import org.sablecc.sablecc.lexer.LexerException;
import org.sablecc.sablecc.node.*;
-@SuppressWarnings({"rawtypes","unchecked","unused"})
+@SuppressWarnings({"deprecation", "rawtypes", "unchecked", "unused"}) // getMapping() is deprecated, but the generated parser still populates it for compatibility
public class Parser implements IParser
{
protected List<Object> nodeList;
@@ -45,8 +45,13 @@ public class Parser implements IParser
this.lexer = lexer;
}
-
+ @Deprecated
private Map<PositionedNode, SourcecodeRange> mapping = new HashMap<PositionedNode, SourcecodeRange>();
+ /**
+ * @deprecated All generated token classes store their own position info.
+ * Use the {@link PositionedNode} or {@link IToken} APIs to access the positions stored in the tokens.
+ */
+ @Deprecated
@Override
public Map<PositionedNode, SourcecodeRange> getMapping() { return this.mapping; }
diff --git a/src/main/resources/org/sablecc/sablecc/lexer.txt b/src/main/resources/org/sablecc/sablecc/lexer.txt
index 44e22ae..5651a1f 100644
--- a/src/main/resources/org/sablecc/sablecc/lexer.txt
+++ b/src/main/resources/org/sablecc/sablecc/lexer.txt
@@ -41,7 +41,7 @@ import de.hhu.stups.sablecc.patch.ITokenListContainer;
import $0$node.*;
-@SuppressWarnings({"unused"})
+@SuppressWarnings({"deprecation", "unused"}) // ITokenListContainer is deprecated, but the generated lexer still implements it for compatibility
public class Lexer implements ITokenListContainer
{
protected Token token;
@@ -61,6 +61,11 @@ public class Lexer implements ITokenListContainer
return nextList;
}
+ /**
+ * @deprecated In the future, the generated lexers will not store all tokens anymore, because this leads to significant memory usage for large inputs.
+ * If you really need this list, please create it yourself, using a custom lexer subclass if necessary.
+ */
+ @Deprecated
@Override
public List<IToken> getTokenList() {
return tokenList;
@@ -70,7 +75,11 @@ public class Lexer implements ITokenListContainer
token = t;
}
-
+ /**
+ * @deprecated In the future, the generated lexers will not store all tokens anymore, because this leads to significant memory usage for large inputs.
+ * If you really need this list, please create it yourself, using a custom lexer subclass if necessary.
+ */
+ @Deprecated
public void setTokenList(final List<IToken> list) {
tokenList = list;
}
diff --git a/src/main/resources/org/sablecc/sablecc/parser.txt b/src/main/resources/org/sablecc/sablecc/parser.txt
index f5b39aa..f7ebae7 100644
--- a/src/main/resources/org/sablecc/sablecc/parser.txt
+++ b/src/main/resources/org/sablecc/sablecc/parser.txt
@@ -30,7 +30,7 @@ import $0$lexer.Lexer;
import $0$lexer.LexerException;
import $0$node.*;
-@SuppressWarnings({"rawtypes","unchecked","unused"})
+@SuppressWarnings({"deprecation", "rawtypes", "unchecked", "unused"}) // getMapping() is deprecated, but the generated parser still populates it for compatibility
public class Parser implements IParser
{
protected List<Object> nodeList;
@@ -53,8 +53,13 @@ public class Parser implements IParser
this.lexer = lexer;
}
-
+ @Deprecated
private Map<PositionedNode, SourcecodeRange> mapping = new HashMap<PositionedNode, SourcecodeRange>();
+ /**
+ * @deprecated All generated token classes store their own position info.
+ * Use the {@link PositionedNode} or {@link IToken} APIs to access the positions stored in the tokens.
+ */
+ @Deprecated
@Override
public Map<PositionedNode, SourcecodeRange> getMapping() { return this.mapping; }
--
GitLab