From 23ddff6eb24d968080242d2c5329463e131ac85e Mon Sep 17 00:00:00 2001 From: chromapid <67613147+chromapid@users.noreply.github.com> Date: Sun, 19 Dec 2021 23:48:59 -0700 Subject: [PATCH] Tests, fixes for constant folding --- .../simon987/mar/server/assembly/Operand.java | 30 ++- .../mar/server/assembly/TokenParser.java | 31 ++- .../mar/server/assembly/TokenTest.java | 185 ++++++++++++++++-- 3 files changed, 218 insertions(+), 28 deletions(-) diff --git a/src/main/java/net/simon987/mar/server/assembly/Operand.java b/src/main/java/net/simon987/mar/server/assembly/Operand.java index d411fdf..7029e78 100755 --- a/src/main/java/net/simon987/mar/server/assembly/Operand.java +++ b/src/main/java/net/simon987/mar/server/assembly/Operand.java @@ -230,7 +230,7 @@ public class Operand { } } - private int parseConstExpression(String text, int line, HashMap labels) + public static int parseConstExpression(String text, int line, HashMap labels) throws AssemblyException { TokenParser parser = new TokenParser(text, line, labels); Stack parseOps = new Stack<>(); @@ -238,7 +238,7 @@ public class Operand { TokenParser.ParseContext context = TokenParser.ParseContext.Value; int lastValue = 0; while (true) { - TokenParser.TokenType ty = parser.GetNextToken(true, context); + TokenParser.TokenType ty = parser.getNextToken(true, context); if (context == TokenParser.ParseContext.Value) { // Parse value if (ty == TokenParser.TokenType.UnaryOperator) { @@ -258,23 +258,38 @@ public class Operand { if (ty == TokenParser.TokenType.EOF || ty == TokenParser.TokenType.GroupOperator) { if (ty == TokenParser.TokenType.GroupOperator && !parser.lastGroup.end) throw new AssemblyException("Unexpected group open", line); - if (closeExpect != -1) throw new AssemblyException("Found empty group", line); + if (closeExpect != -1) { + if (ty == TokenParser.TokenType.EOF) + throw new AssemblyException("Unclosed group", line); + else if (closeExpect != parser.lastGroup.groupType) + throw new AssemblyException("Unmatched group ends", line); + else { + closeExpect = -1; + continue; + } + } + + boolean completed = false; //Evaluation chain while (!parseOps.isEmpty()) { - ParseOperator op = parseOps.peek(); + ParseOperator op = parseOps.pop(); if (op.closeExpect != -1) { if (ty == TokenParser.TokenType.EOF) throw new AssemblyException("Unclosed group", line); else if (op.closeExpect != parser.lastGroup.groupType) throw new AssemblyException("Unmatched group ends", line); lastValue = op.apply(lastValue); - parseOps.pop(); + completed = true; break; } lastValue = op.apply(lastValue); - parseOps.pop(); } - if (parseOps.isEmpty() && ty == TokenParser.TokenType.EOF) return lastValue; + if (!completed) { + if (ty == TokenParser.TokenType.EOF) return lastValue; + else if (parser.lastGroup.groupType != -1) + throw new AssemblyException("Unexpected group close", line); + } + } else if (ty == TokenParser.TokenType.BinaryOperator) { TokenParser.BinaryOperatorType bop = parser.lastBinary; @@ -287,6 +302,7 @@ public class Operand { } parseOps.push(new ParseOperatorBinary(closeExpect, bop, lastValue)); closeExpect = -1; + context = TokenParser.ParseContext.Value; } else throw new AssemblyException("Modifier or end not found", line); } diff --git a/src/main/java/net/simon987/mar/server/assembly/TokenParser.java b/src/main/java/net/simon987/mar/server/assembly/TokenParser.java index 668e79c..2423542 100644 --- a/src/main/java/net/simon987/mar/server/assembly/TokenParser.java +++ b/src/main/java/net/simon987/mar/server/assembly/TokenParser.java @@ -6,6 +6,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import net.simon987.mar.server.assembly.exception.AssemblyException; +import org.apache.velocity.runtime.directive.Parse; public class TokenParser { @@ -151,17 +152,17 @@ public class TokenParser { } private static final Pattern BINARY_OPERATOR_PATTERN = Pattern.compile("(<<|>>|[\\-+*/<>&|^])"); - private static final Pattern UNARY_OPERATOR_PATTERN = Pattern.compile("([\\-!])"); + private static final Pattern UNARY_OPERATOR_PATTERN = Pattern.compile("([\\-~])"); private static final Pattern GROUP_OPERATOR_PATTERN = Pattern.compile("([()])"); - private static final Pattern IDENTIFIER_PATTERN = Pattern.compile("([\\w])"); + private static final Pattern IDENTIFIER_PATTERN = Pattern.compile("([\\w]+)"); private static final Pattern NUMBER_PATTERN_16 = Pattern.compile("0[xX]([\\da-fA-F]+)(?![\\w\\d])"); private static final Pattern NUMBER_PATTERN_8 = Pattern.compile("0[oO]([0-7]+)(?![\\w\\d])"); private static final Pattern NUMBER_PATTERN_2 = Pattern.compile("0[bB]([01]+)(?![\\w\\d])"); private static final Pattern NUMBER_PATTERN_10 = Pattern.compile("(\\d+)(?![\\w\\d])"); private static final Pattern NUMBER_PATTERN_START = Pattern.compile("(\\d)"); + private static final Pattern RESET_PATTERN = Pattern.compile("[^\\w\\d]"); private static final Pattern SPACE_PATTERN = Pattern.compile("[^\\S\\n]+"); - /** * @param sequence The characters to parse * @param start The index of the first character to be parsed @@ -203,7 +204,7 @@ public class TokenParser { * @throws AssemblyException if an unrecognized token is found, * or if the found token is not supported in the current context. */ - public TokenType GetNextToken(boolean eatSpace, ParseContext context) throws AssemblyException { + public TokenType getNextToken(boolean eatSpace, ParseContext context) throws AssemblyException { if (start >= end) return TokenType.EOF; matcher.region(start, end); if (matcher.usePattern(SPACE_PATTERN).lookingAt()) { @@ -214,12 +215,13 @@ public class TokenParser { } matcher.usePattern(GROUP_OPERATOR_PATTERN); if (matcher.lookingAt()) { - start = matcher.end(); + start = matcher.end(1); String symbol = matcher.group(1); lastGroup = GroupOperatorType.stringMap.get(symbol); // Should never happen unless the regex does not agree with GroupOperatorType. if (lastGroup == null) throw new AssemblyException("Group operator not supported", line); + return TokenType.GroupOperator; } if (context == ParseContext.TackOn) { @@ -236,17 +238,23 @@ public class TokenParser { } else { if (matcher.usePattern(NUMBER_PATTERN_START).lookingAt()) { + try { if (matcher.usePattern(NUMBER_PATTERN_10).lookingAt()) lastInt = Integer.parseInt(matcher.group(1), 10); else if (matcher.usePattern(NUMBER_PATTERN_16).lookingAt()) - lastInt = Integer.parseInt( matcher.group(1), 16); + lastInt = Integer.parseInt(matcher.group(1), 16); else if (matcher.usePattern(NUMBER_PATTERN_2).lookingAt()) lastInt = Integer.parseInt(matcher.group(1), 2); else if (matcher.usePattern(NUMBER_PATTERN_8).lookingAt()) lastInt = Integer.parseInt(matcher.group(1), 8); - else throw new AssemblyException("Invalid number found.", line); + else { + if (matcher.usePattern(RESET_PATTERN).find()) start = matcher.start(); + else start = end; + throw new AssemblyException("Invalid number found.", line); + } } catch (NumberFormatException ex) { + start = matcher.end(1); throw new AssemblyException("Number parsing failed", line); } start = matcher.end(1); @@ -254,25 +262,28 @@ public class TokenParser { return TokenType.Constant; } if (matcher.usePattern(IDENTIFIER_PATTERN).lookingAt()) { - start = matcher.end(); + start = matcher.end(1); String identifier = matcher.group(1); Character val = labels.get(identifier); if (val == null) throw new AssemblyException("Unknown label found", line); + lastInt = val; return TokenType.Constant; } matcher.usePattern(UNARY_OPERATOR_PATTERN); if (matcher.lookingAt()) { - start = matcher.end(); + start = matcher.end(1); String symbol = matcher.group(1); lastUnary = UnaryOperatorType.stringMap.get(symbol); // Should never happen unless the regex does not agree with UnaryOperatorType. if (lastUnary == null) throw new AssemblyException("Unary operator not supported", line); - return TokenType.BinaryOperator; + return TokenType.UnaryOperator; } } + if (matcher.usePattern(RESET_PATTERN).find()) start = matcher.end(); + else start = end; throw new AssemblyException("Invalid token found", line); } diff --git a/src/test/java/net/simon987/mar/server/assembly/TokenTest.java b/src/test/java/net/simon987/mar/server/assembly/TokenTest.java index 7fc7754..c55ee34 100644 --- a/src/test/java/net/simon987/mar/server/assembly/TokenTest.java +++ b/src/test/java/net/simon987/mar/server/assembly/TokenTest.java @@ -1,25 +1,188 @@ package net.simon987.mar.server.assembly; +import net.simon987.mar.server.assembly.exception.AssemblyException; import org.junit.Test; import java.util.HashMap; +import java.util.Map; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; public class TokenTest { @Test - public void numberTokenParsing() throws Exception { - TokenParser parser = new TokenParser("27 0x27 0o27 0b11011", 0, new HashMap<>()); - assertEquals(TokenParser.TokenType.Constant, parser.GetNextToken(true, TokenParser.ParseContext.Value)); - assertEquals(27, parser.lastInt); + public void constantTokenParsing() throws Exception { + Map labels = new HashMap<>(); + labels.put("alpha", (char)29); + labels.put("beta", (char)28); + labels.put("gamma", (char)-5); + labels.put("epsilon", (char)0); + TokenParser parser = new TokenParser( + " 27 0x27 0o27 0b11011 alpha beta gamma delta epsilon 0 1a 0xG 0o8 0b2", + 0, labels); - assertEquals(TokenParser.TokenType.Constant, parser.GetNextToken(true, TokenParser.ParseContext.Value)); - assertEquals(39, parser.lastInt); + assertEquals(TokenParser.TokenType.Space, parser.getNextToken(false, TokenParser.ParseContext.Value)); + assertEquals(TokenParser.TokenType.Constant, parser.getNextToken(false, TokenParser.ParseContext.Value)); + assertEquals((char)27, parser.lastInt); + assertEquals(TokenParser.TokenType.Space, parser.getNextToken(false, TokenParser.ParseContext.TackOn)); + assertEquals(TokenParser.TokenType.Constant, parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals((char)39, parser.lastInt); + assertEquals(TokenParser.TokenType.Constant, parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals((char)23, parser.lastInt); + assertEquals(TokenParser.TokenType.Constant, parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals((char)27, parser.lastInt); - assertEquals(TokenParser.TokenType.Constant, parser.GetNextToken(true, TokenParser.ParseContext.Value)); - assertEquals(23, parser.lastInt); + // Labels + assertEquals(TokenParser.TokenType.Constant, parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals((char)29, parser.lastInt); + assertEquals(TokenParser.TokenType.Constant, parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals((char)28, parser.lastInt); + assertEquals(TokenParser.TokenType.Constant, parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals((char)-5, parser.lastInt); + try { + parser.getNextToken(true, TokenParser.ParseContext.Value); + fail(); + } catch (AssemblyException ignore) {} + try { + parser.getNextToken(true, TokenParser.ParseContext.TackOn); + fail(); + } catch (AssemblyException ignore) {} + try { + parser.getNextToken(true, TokenParser.ParseContext.TackOn); + fail(); + } catch (AssemblyException ignore) {} + try { + parser.getNextToken(true, TokenParser.ParseContext.Value); + fail(); + } catch (AssemblyException ignore) {} + try { + parser.getNextToken(true, TokenParser.ParseContext.Value); + fail(); + } catch (AssemblyException ignore) {} + try { + parser.getNextToken(true, TokenParser.ParseContext.Value); + fail(); + } catch (AssemblyException ignore) {} + try { + parser.getNextToken(true, TokenParser.ParseContext.Value); + fail(); + } catch (AssemblyException ignore) {} + } + @Test + public void operatorTokenParsing() throws Exception { + String ops = "+~-"; + TokenParser parser = new TokenParser(ops, 0, new HashMap<>()); + assertEquals(TokenParser.TokenType.BinaryOperator, + parser.getNextToken(true, TokenParser.ParseContext.TackOn)); + assertEquals(TokenParser.BinaryOperatorType.Add, parser.lastBinary); + try { + parser.getNextToken(true, TokenParser.ParseContext.TackOn); + fail(); + } catch (AssemblyException ignore) {} + assertEquals(TokenParser.TokenType.BinaryOperator, + parser.getNextToken(true, TokenParser.ParseContext.TackOn)); + assertEquals(TokenParser.BinaryOperatorType.Sub, parser.lastBinary); - assertEquals(TokenParser.TokenType.Constant, parser.GetNextToken(true, TokenParser.ParseContext.Value)); - assertEquals(27, parser.lastInt); + + parser = new TokenParser(ops, 0, new HashMap<>()); + try { + parser.getNextToken(true, TokenParser.ParseContext.Value); + fail(); + } catch (AssemblyException ignore) {} + + assertEquals(TokenParser.TokenType.UnaryOperator, + parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals(TokenParser.UnaryOperatorType.Not, parser.lastUnary); + assertEquals(TokenParser.TokenType.UnaryOperator, + parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals(TokenParser.UnaryOperatorType.Neg, parser.lastUnary); + parser = new TokenParser("()", 0, new HashMap<>()); + assertEquals(TokenParser.TokenType.GroupOperator, + parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals(TokenParser.GroupOperatorType.GroupStart, parser.lastGroup); + assertEquals(TokenParser.TokenType.GroupOperator, + parser.getNextToken(true, TokenParser.ParseContext.Value)); + assertEquals(TokenParser.GroupOperatorType.GroupEnd, parser.lastGroup); + } + + @Test + public void parseTest() throws AssemblyException { + assertEquals(1, Operand.parseConstExpression( + "1", + 0, new HashMap<>()) + ); + assertEquals(2, Operand.parseConstExpression( + "1 + 1", + 0, new HashMap<>()) + ); + assertEquals(82, Operand.parseConstExpression( + "10 + 9 * 8", + 0, new HashMap<>()) + ); + assertEquals(98, Operand.parseConstExpression( + "10 * 9 + 8", + 0, new HashMap<>()) + ); + assertEquals(2, Operand.parseConstExpression( + "(1 + 1)", + 0, new HashMap<>()) + ); + assertEquals(152, Operand.parseConstExpression( + "((10 + 9)) * 8", + 0, new HashMap<>()) + ); + assertEquals(170, Operand.parseConstExpression( + "(10 * ((9 + 8)))", + 0, new HashMap<>()) + ); + assertEquals((char)-170, Operand.parseConstExpression( + "(-10 * ((9 + 8)))", + 0, new HashMap<>()) + ); + assertEquals(2, Operand.parseConstExpression( + "(-3 + 5)", + 0, new HashMap<>()) + ); + try { + Operand.parseConstExpression( + "(1", + 0, new HashMap<>() + ); + fail(); + } catch (AssemblyException ignore) {} + try { + Operand.parseConstExpression( + "1)", + 0, new HashMap<>() + ); + fail(); + } catch (AssemblyException ignore) {} + try { + Operand.parseConstExpression( + "(1+1", + 0, new HashMap<>() + ); + fail(); + } catch (AssemblyException ignore) {} + try { + Operand.parseConstExpression( + "1+1)", + 0, new HashMap<>() + ); + fail(); + } catch (AssemblyException ignore) {} + try { + Operand.parseConstExpression( + "((1+1)", + 0, new HashMap<>() + ); + fail(); + } catch (AssemblyException ignore) {} + try { + Operand.parseConstExpression( + "(1+1))", + 0, new HashMap<>() + ); + fail(); + } catch (AssemblyException ignore) {} } }