Skip to content
Snippets Groups Projects
Commit 381f0aaf authored by dgelessus's avatar dgelessus
Browse files

Consistently use UTF-8 instead of platform default charset

Fixes platform-dependent behavior when using non-ASCII characters in
grammar source files.
parent d038f909
No related branches found
No related tags found
No related merge requests found
Pipeline #113517 passed
...@@ -10,6 +10,7 @@ package org.sablecc.sablecc; ...@@ -10,6 +10,7 @@ package org.sablecc.sablecc;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
class DisplayLicense class DisplayLicense
{ {
...@@ -20,7 +21,7 @@ class DisplayLicense ...@@ -20,7 +21,7 @@ class DisplayLicense
BufferedReader in = BufferedReader in =
new BufferedReader( new BufferedReader(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("LICENSE"))); getClass().getResourceAsStream("LICENSE"), StandardCharsets.UTF_8));
System.out.println("---- FILE: LICENSE ----"); System.out.println("---- FILE: LICENSE ----");
String s; String s;
...@@ -37,7 +38,7 @@ class DisplayLicense ...@@ -37,7 +38,7 @@ class DisplayLicense
in = in =
new BufferedReader( new BufferedReader(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("AUTHORS"))); getClass().getResourceAsStream("AUTHORS"), StandardCharsets.UTF_8));
while((s = in.readLine()) != null) while((s = in.readLine()) != null)
{ {
...@@ -52,7 +53,7 @@ class DisplayLicense ...@@ -52,7 +53,7 @@ class DisplayLicense
in = in =
new BufferedReader( new BufferedReader(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("COPYING-LESSER"))); getClass().getResourceAsStream("COPYING-LESSER"), StandardCharsets.UTF_8));
while((s = in.readLine()) != null) while((s = in.readLine()) != null)
{ {
......
...@@ -9,9 +9,10 @@ package org.sablecc.sablecc; ...@@ -9,9 +9,10 @@ package org.sablecc.sablecc;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
...@@ -37,7 +38,7 @@ public class GenAlts extends DepthFirstAdapter ...@@ -37,7 +38,7 @@ public class GenAlts extends DepthFirstAdapter
{ {
macros = new MacroExpander( macros = new MacroExpander(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("alternatives.txt"))); getClass().getResourceAsStream("alternatives.txt"), StandardCharsets.UTF_8));
} }
catch(IOException e) catch(IOException e)
{ {
...@@ -112,7 +113,7 @@ public class GenAlts extends DepthFirstAdapter ...@@ -112,7 +113,7 @@ public class GenAlts extends DepthFirstAdapter
{ {
String name = ast_ids.ast_names.get(node); String name = ast_ids.ast_names.get(node);
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, name + ".java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, name + ".java").toPath(), StandardCharsets.UTF_8))
{ {
boolean hasOperator = false; boolean hasOperator = false;
boolean hasList = false; boolean hasList = false;
......
...@@ -9,9 +9,10 @@ package org.sablecc.sablecc; ...@@ -9,9 +9,10 @@ package org.sablecc.sablecc;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.ListIterator; import java.util.ListIterator;
...@@ -39,7 +40,7 @@ public class GenAnalyses extends DepthFirstAdapter ...@@ -39,7 +40,7 @@ public class GenAnalyses extends DepthFirstAdapter
{ {
macros = new MacroExpander( macros = new MacroExpander(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("analyses.txt"))); getClass().getResourceAsStream("analyses.txt"), StandardCharsets.UTF_8));
} }
catch(IOException e) catch(IOException e)
{ {
...@@ -148,7 +149,7 @@ public class GenAnalyses extends DepthFirstAdapter ...@@ -148,7 +149,7 @@ public class GenAnalyses extends DepthFirstAdapter
public void createAnalysis() public void createAnalysis()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "Analysis.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "Analysis.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "AnalysisHeader", new String[] {ast_ids.astIds.pkgNameDot}); macros.apply(file, "AnalysisHeader", new String[] {ast_ids.astIds.pkgNameDot});
...@@ -182,7 +183,7 @@ public class GenAnalyses extends DepthFirstAdapter ...@@ -182,7 +183,7 @@ public class GenAnalyses extends DepthFirstAdapter
public void createAnalysisAdapter() public void createAnalysisAdapter()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "AnalysisAdapter.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "AnalysisAdapter.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "AnalysisAdapterHeader", new String[] {ast_ids.astIds.pkgNameDot}); macros.apply(file, "AnalysisAdapterHeader", new String[] {ast_ids.astIds.pkgNameDot});
...@@ -214,7 +215,7 @@ public class GenAnalyses extends DepthFirstAdapter ...@@ -214,7 +215,7 @@ public class GenAnalyses extends DepthFirstAdapter
public void createDepthFirstAdapter() public void createDepthFirstAdapter()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "DepthFirstAdapter.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "DepthFirstAdapter.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "DepthFirstAdapterHeader", new String[] {ast_ids.astIds.pkgNameDot, mainProduction}); macros.apply(file, "DepthFirstAdapterHeader", new String[] {ast_ids.astIds.pkgNameDot, mainProduction});
...@@ -263,7 +264,7 @@ public class GenAnalyses extends DepthFirstAdapter ...@@ -263,7 +264,7 @@ public class GenAnalyses extends DepthFirstAdapter
public void createReversedDepthFirstAdapter() public void createReversedDepthFirstAdapter()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "ReversedDepthFirstAdapter.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "ReversedDepthFirstAdapter.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "ReversedDepthFirstAdapterHeader", new String[] {ast_ids.astIds.pkgNameDot, mainProduction}); macros.apply(file, "ReversedDepthFirstAdapterHeader", new String[] {ast_ids.astIds.pkgNameDot, mainProduction});
......
...@@ -12,9 +12,10 @@ import java.io.BufferedWriter; ...@@ -12,9 +12,10 @@ import java.io.BufferedWriter;
import java.io.DataOutputStream; import java.io.DataOutputStream;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
...@@ -41,7 +42,7 @@ public class GenLexer extends AnalysisAdapter ...@@ -41,7 +42,7 @@ public class GenLexer extends AnalysisAdapter
{ {
macros = new MacroExpander( macros = new MacroExpander(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("lexer.txt"))); getClass().getResourceAsStream("lexer.txt"), StandardCharsets.UTF_8));
} }
catch(IOException e) catch(IOException e)
{ {
...@@ -111,7 +112,7 @@ public class GenLexer extends AnalysisAdapter ...@@ -111,7 +112,7 @@ public class GenLexer extends AnalysisAdapter
private void createLexerException() private void createLexerException()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "LexerException.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "LexerException.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "LexerException", new String[] {ids.pkgNameDot}); macros.apply(file, "LexerException", new String[] {ids.pkgNameDot});
} }
...@@ -124,7 +125,7 @@ public class GenLexer extends AnalysisAdapter ...@@ -124,7 +125,7 @@ public class GenLexer extends AnalysisAdapter
private void createLexer() private void createLexer()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "Lexer.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "Lexer.java").toPath(), StandardCharsets.UTF_8))
{ {
String startState = "INITIAL"; String startState = "INITIAL";
if(ids.stateList.size() > 0) if(ids.stateList.size() > 0)
......
...@@ -12,9 +12,10 @@ import java.io.BufferedWriter; ...@@ -12,9 +12,10 @@ import java.io.BufferedWriter;
import java.io.DataOutputStream; import java.io.DataOutputStream;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Deque; import java.util.Deque;
import java.util.HashMap; import java.util.HashMap;
...@@ -107,7 +108,7 @@ public class GenParser extends DepthFirstAdapter ...@@ -107,7 +108,7 @@ public class GenParser extends DepthFirstAdapter
{ {
macros = new MacroExpander( macros = new MacroExpander(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("parser.txt"))); getClass().getResourceAsStream("parser.txt"), StandardCharsets.UTF_8));
} }
catch(IOException e) catch(IOException e)
{ {
...@@ -366,7 +367,7 @@ public class GenParser extends DepthFirstAdapter ...@@ -366,7 +367,7 @@ public class GenParser extends DepthFirstAdapter
//Parser.java Generation //Parser.java Generation
private void createParser() private void createParser()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "Parser.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "Parser.java").toPath(), StandardCharsets.UTF_8))
{ {
Symbol[] terminals = Symbol.terminals(); Symbol[] terminals = Symbol.terminals();
Symbol[] nonterminals = Symbol.nonterminals(); Symbol[] nonterminals = Symbol.nonterminals();
...@@ -726,7 +727,7 @@ public class GenParser extends DepthFirstAdapter ...@@ -726,7 +727,7 @@ public class GenParser extends DepthFirstAdapter
private void createTokenIndex() private void createTokenIndex()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "TokenIndex.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "TokenIndex.java").toPath(), StandardCharsets.UTF_8))
{ {
Symbol[] terminals = Symbol.terminals(); Symbol[] terminals = Symbol.terminals();
...@@ -748,7 +749,7 @@ public class GenParser extends DepthFirstAdapter ...@@ -748,7 +749,7 @@ public class GenParser extends DepthFirstAdapter
private void createParserException() private void createParserException()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "ParserException.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "ParserException.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "ParserException", new String[] {ids.pkgNameDot}); macros.apply(file, "ParserException", new String[] {ids.pkgNameDot});
} }
...@@ -761,7 +762,7 @@ public class GenParser extends DepthFirstAdapter ...@@ -761,7 +762,7 @@ public class GenParser extends DepthFirstAdapter
private void createState() private void createState()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "State.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "State.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "State", new String[] {ids.pkgNameDot}); macros.apply(file, "State", new String[] {ids.pkgNameDot});
} }
......
...@@ -9,9 +9,10 @@ package org.sablecc.sablecc; ...@@ -9,9 +9,10 @@ package org.sablecc.sablecc;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import org.sablecc.sablecc.analysis.DepthFirstAdapter; import org.sablecc.sablecc.analysis.DepthFirstAdapter;
import org.sablecc.sablecc.node.AAstProd; import org.sablecc.sablecc.node.AAstProd;
...@@ -30,7 +31,7 @@ public class GenProds extends DepthFirstAdapter ...@@ -30,7 +31,7 @@ public class GenProds extends DepthFirstAdapter
{ {
macros = new MacroExpander( macros = new MacroExpander(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("productions.txt"))); getClass().getResourceAsStream("productions.txt"), StandardCharsets.UTF_8));
} }
catch(IOException e) catch(IOException e)
{ {
...@@ -58,7 +59,7 @@ public class GenProds extends DepthFirstAdapter ...@@ -58,7 +59,7 @@ public class GenProds extends DepthFirstAdapter
private void createProduction(String name) private void createProduction(String name)
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, name + ".java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, name + ".java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "Production", new String[] {ast_ids.astIds.pkgNameDot, name}); macros.apply(file, "Production", new String[] {ast_ids.astIds.pkgNameDot, name});
} }
...@@ -71,7 +72,7 @@ public class GenProds extends DepthFirstAdapter ...@@ -71,7 +72,7 @@ public class GenProds extends DepthFirstAdapter
private void createAlternative(String name, String macro, String[] arg) private void createAlternative(String name, String macro, String[] arg)
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, name + ".java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, name + ".java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, macro, arg); macros.apply(file, macro, arg);
} }
......
...@@ -9,9 +9,10 @@ package org.sablecc.sablecc; ...@@ -9,9 +9,10 @@ package org.sablecc.sablecc;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.List; import java.util.List;
import org.sablecc.sablecc.analysis.DepthFirstAdapter; import org.sablecc.sablecc.analysis.DepthFirstAdapter;
...@@ -34,7 +35,7 @@ public class GenTokens extends DepthFirstAdapter ...@@ -34,7 +35,7 @@ public class GenTokens extends DepthFirstAdapter
{ {
macros = new MacroExpander( macros = new MacroExpander(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("tokens.txt"))); getClass().getResourceAsStream("tokens.txt"), StandardCharsets.UTF_8));
} }
catch(IOException e) catch(IOException e)
{ {
...@@ -93,7 +94,7 @@ public class GenTokens extends DepthFirstAdapter ...@@ -93,7 +94,7 @@ public class GenTokens extends DepthFirstAdapter
} }
} }
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, name + ".java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, name + ".java").toPath(), StandardCharsets.UTF_8))
{ {
if(text == null) if(text == null)
{ {
......
...@@ -9,9 +9,10 @@ package org.sablecc.sablecc; ...@@ -9,9 +9,10 @@ package org.sablecc.sablecc;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import org.sablecc.sablecc.analysis.DepthFirstAdapter; import org.sablecc.sablecc.analysis.DepthFirstAdapter;
import org.sablecc.sablecc.node.AAstProd; import org.sablecc.sablecc.node.AAstProd;
...@@ -32,7 +33,7 @@ public class GenUtils extends DepthFirstAdapter ...@@ -32,7 +33,7 @@ public class GenUtils extends DepthFirstAdapter
{ {
macros = new MacroExpander( macros = new MacroExpander(
new InputStreamReader( new InputStreamReader(
getClass().getResourceAsStream("utils.txt"))); getClass().getResourceAsStream("utils.txt"), StandardCharsets.UTF_8));
} }
catch(IOException e) catch(IOException e)
{ {
...@@ -88,7 +89,7 @@ public class GenUtils extends DepthFirstAdapter ...@@ -88,7 +89,7 @@ public class GenUtils extends DepthFirstAdapter
public void createStart() public void createStart()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "Start.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "Start.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "Start", new String[] {ast_ids.astIds.pkgNameDot, mainProduction, GenAlts.nodeName(mainProduction)}); macros.apply(file, "Start", new String[] {ast_ids.astIds.pkgNameDot, mainProduction, GenAlts.nodeName(mainProduction)});
} }
...@@ -101,7 +102,7 @@ public class GenUtils extends DepthFirstAdapter ...@@ -101,7 +102,7 @@ public class GenUtils extends DepthFirstAdapter
public void createEOF() public void createEOF()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "EOF.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "EOF.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "EOF", new String[] {ast_ids.astIds.pkgNameDot}); macros.apply(file, "EOF", new String[] {ast_ids.astIds.pkgNameDot});
} }
...@@ -114,7 +115,7 @@ public class GenUtils extends DepthFirstAdapter ...@@ -114,7 +115,7 @@ public class GenUtils extends DepthFirstAdapter
public void createNode() public void createNode()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "Node.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "Node.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "Node", new String[] {ast_ids.astIds.pkgNameDot}); macros.apply(file, "Node", new String[] {ast_ids.astIds.pkgNameDot});
} }
...@@ -127,7 +128,7 @@ public class GenUtils extends DepthFirstAdapter ...@@ -127,7 +128,7 @@ public class GenUtils extends DepthFirstAdapter
public void createToken() public void createToken()
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, "Token.java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, "Token.java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, "Token", new String[] {ast_ids.astIds.pkgNameDot}); macros.apply(file, "Token", new String[] {ast_ids.astIds.pkgNameDot});
} }
...@@ -140,7 +141,7 @@ public class GenUtils extends DepthFirstAdapter ...@@ -140,7 +141,7 @@ public class GenUtils extends DepthFirstAdapter
public void create(String cls) public void create(String cls)
{ {
try(BufferedWriter file = new BufferedWriter(new FileWriter(new File(pkgDir, cls + ".java")))) try(BufferedWriter file = Files.newBufferedWriter(new File(pkgDir, cls + ".java").toPath(), StandardCharsets.UTF_8))
{ {
macros.apply(file, cls, new String[] {ast_ids.astIds.pkgNameDot}); macros.apply(file, cls, new String[] {ast_ids.astIds.pkgNameDot});
} }
......
...@@ -7,10 +7,12 @@ ...@@ -7,10 +7,12 @@
package org.sablecc.sablecc; package org.sablecc.sablecc;
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.FileReader;
import java.io.IOException; import java.io.IOException;
import java.io.PushbackReader; import java.io.PushbackReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
...@@ -186,13 +188,14 @@ public class SableCC { ...@@ -186,13 +188,14 @@ public class SableCC {
System.out.println("\n -- Generating parser for " + in.getName() System.out.println("\n -- Generating parser for " + in.getName()
+ " in " + dir.getPath()); + " in " + dir.getPath());
FileReader temp = new FileReader(in);
// Build the AST // Build the AST
Start tree = new Parser(new Lexer(new PushbackReader( Start tree;
temp = new FileReader(in), 1000))).parse(); try (
BufferedReader temp = Files.newBufferedReader(in.toPath(), StandardCharsets.UTF_8);
temp.close(); PushbackReader reader = new PushbackReader(temp, 1000);
) {
tree = new Parser(new Lexer(reader)).parse();
}
boolean hasTransformations = false; boolean hasTransformations = false;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment