1 module sdlite.parser; 2 3 import sdlite.ast; 4 import sdlite.internal : MultiAppender; 5 import sdlite.lexer; 6 7 void parseSDLDocument(alias NodeHandler, R)(R input, string filename) 8 { 9 import std.algorithm.comparison : among; 10 import std.algorithm.iteration : filter; 11 12 auto tokens = lexSDLang(input, filename) 13 .filter!(t => t.type != TokenType.comment); 14 15 ParserContext ctx; 16 17 parseNodes!NodeHandler(tokens, ctx, 0); 18 19 while (!tokens.empty) { 20 if (!tokens.front.type.among(TokenType.eof, TokenType.comment)) 21 throw new Exception("Expected end of file"); 22 tokens.popFront(); 23 } 24 } 25 26 unittest { 27 void test(string sdl, SDLNode[] expected) 28 { 29 SDLNode[] result; 30 parseSDLDocument!((n) { result ~= n; })(sdl, "test"); 31 import std.conv : to; 32 assert(result == expected, result.to!string); 33 } 34 35 test("foo", [SDLNode("foo")]); 36 test("foo:bar", [SDLNode("foo:bar")]); 37 test("foo 123", [SDLNode("foo", 38 [SDLValue.int_(123)])]); 39 test("foo null\nbar", [SDLNode("foo", [SDLValue.null_]), SDLNode("bar")]); 40 test("foo null;bar", [SDLNode("foo", [SDLValue.null_]), SDLNode("bar")]); 41 test("foo {\n}\n\nbar", [SDLNode("foo"), SDLNode("bar")]); 42 test("foo bar=123", [SDLNode("foo", null, 43 [SDLAttribute("bar", SDLValue.int_(123))])]); 44 test("foo 42 bar=123", [SDLNode("foo", 45 [SDLValue.int_(42)], 46 [SDLAttribute("bar", SDLValue.int_(123))])]); 47 test("foo {\nbar\n}", [SDLNode("foo", null, null, [SDLNode("bar")])]); 48 test("\nfoo", [SDLNode("foo")]); 49 } 50 51 private void parseNodes(alias NodeHandler, R)(ref R tokens, ref ParserContext ctx, size_t depth) 52 { 53 import std.algorithm.comparison : among; 54 55 while (!tokens.empty && tokens.front.type.among(TokenType.eol, TokenType.semicolon)) 56 tokens.popFront(); 57 58 while (!tokens.empty && !tokens.front.type.among(TokenType.eof, TokenType.blockClose)) { 59 auto n = tokens.parseNode(ctx, depth); 60 NodeHandler(n); 61 62 while (!tokens.empty && tokens.front.type.among(TokenType.eol, TokenType.semicolon)) 63 tokens.popFront(); 64 } 65 } 66 67 private SDLNode parseNode(R)(ref R tokens, ref ParserContext ctx, size_t depth) 68 { 69 SDLNode ret; 70 71 bool require_parameters = false; 72 73 auto n = tokens.parseQualifiedName(false, ctx); 74 if (n is null) { 75 n = "content"; 76 require_parameters = true; 77 } 78 79 ret.qualifiedName = n; 80 ret.values = tokens.parseValues(ctx); 81 import std.conv; 82 if (require_parameters && ret.values.length == 0) 83 throw new Exception("Expected values for anonymous node"~tokens.front.to!string); 84 ret.attributes = tokens.parseAttributes(ctx); 85 86 if (!tokens.empty && tokens.front.type == TokenType.blockOpen) { 87 tokens.popFront(); 88 tokens.skipToken(TokenType.eol); 89 90 if (ctx.nodeAppender.length <= depth) 91 ctx.nodeAppender.length = depth+1; 92 tokens.parseNodes!((ref n) { ctx.nodeAppender[depth].put(n); })(ctx, depth+1); 93 ret.children = ctx.nodeAppender[depth].extractArray; 94 95 if (tokens.empty || tokens.front.type != TokenType.blockClose) 96 throw new Exception("Expected }"); 97 98 tokens.popFront(); 99 if (!tokens.empty && tokens.front.type != TokenType.eof) 100 tokens.skipToken(TokenType.eol); 101 } 102 103 return ret; 104 } 105 106 private SDLAttribute[] parseAttributes(R)(ref R tokens, ref ParserContext ctx) 107 { 108 while (!tokens.empty && tokens.front.type == TokenType.identifier) { 109 SDLAttribute att; 110 att.qualifiedName = tokens.parseQualifiedName(true, ctx); 111 tokens.skipToken(TokenType.assign); 112 if (!tokens.parseValue(att.value, ctx)) 113 throw new Exception("Expected attribute value"); 114 ctx.attributeAppender.put(att); 115 } 116 117 return ctx.attributeAppender.extractArray; 118 } 119 120 private SDLValue[] parseValues(R)(ref R tokens, ref ParserContext ctx) 121 { 122 while (!tokens.empty) { 123 SDLValue v; 124 if (!tokens.parseValue(v, ctx)) 125 break; 126 ctx.valueAppender.put(v); 127 } 128 129 return ctx.valueAppender.extractArray; 130 } 131 132 private bool parseValue(R)(ref R tokens, ref SDLValue dst, ref ParserContext ctx) 133 { 134 switch (tokens.front.type) { 135 default: return false; 136 case TokenType.null_: 137 case TokenType.text: 138 case TokenType.binary: 139 case TokenType.number: 140 case TokenType.boolean: 141 case TokenType.dateTime: 142 case TokenType.date: 143 case TokenType.duration: 144 dst = sdlite.lexer.parseValue!(typeof(tokens.front).SourceRange)(tokens.front, ctx.charAppender, ctx.bytesAppender); 145 tokens.popFront(); 146 return true; 147 } 148 } 149 150 private string parseQualifiedName(R)(ref R tokens, bool required, ref ParserContext ctx) 151 { 152 import std.array : array; 153 import std.exception : assumeUnique; 154 import std.range : chain; 155 import std.utf : byCodeUnit; 156 157 if (tokens.front.type != TokenType.identifier) { 158 if (required) throw new Exception("Expected identifier"); 159 else return null; 160 } 161 162 foreach (ch; tokens.front.text) 163 ctx.charAppender.put(ch); 164 tokens.popFront(); 165 166 if (!tokens.empty && tokens.front.type == TokenType.namespace) { 167 tokens.popFront(); 168 if (tokens.empty || tokens.front.type != TokenType.identifier) 169 throw new Exception("Expected identifier"); 170 171 ctx.charAppender.put(':'); 172 foreach (ch; tokens.front.text) 173 ctx.charAppender.put(ch); 174 tokens.popFront(); 175 } 176 177 return ctx.charAppender.extractArray; 178 } 179 180 private void skipToken(R)(ref R tokens, scope TokenType[] allowed_types...) 181 { 182 import std.algorithm.searching : canFind; 183 import std.format : format; 184 185 if (tokens.empty) throw new Exception("Unexpected end of file"); 186 if (!allowed_types.canFind(tokens.front.type)) 187 throw new Exception(format("Unexpected token at line %s, %s, expected any of %s", tokens.front.location.line+1, tokens.front.type, allowed_types)); 188 189 tokens.popFront(); 190 } 191 192 private struct ParserContext { 193 MultiAppender!SDLValue valueAppender; 194 MultiAppender!SDLAttribute attributeAppender; 195 MultiAppender!(immutable(char)) charAppender; 196 MultiAppender!(immutable(ubyte)) bytesAppender; 197 MultiAppender!SDLNode[] nodeAppender; // one for each recursion depth 198 }