3 * Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4 * Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
6 * Copyright (C) 2012 - 2016 - Scilab Enterprises
8 * This file is hereby licensed under the terms of the GNU GPL v2.0,
9 * pursuant to article 5.3.4 of the CeCILL v.2.1.
10 * This file was originally licensed under the terms of the CeCILL v2.1,
11 * and continues to be available under such terms.
12 * For more information, see the COPYING file which you should have received
13 * along with this program.
21 #include "parser_private.hxx"
23 #include "context.hxx"
27 #include "charEncoding.h"
28 #include "sci_malloc.h"
31 static std::stack<int> paren_levels;
33 static int comment_level = 0;
34 static int last_token = 0;
35 static int linebreak_stored_token = 0;
36 static bool linebreak_stored_space = FALSE;
37 static int exit_status = PARSE_ERROR;
38 static int str_opener_column = 0;
39 static std::string current_file;
40 static std::string program_name;
42 static std::string pstBuffer;
44 extern void yyerror(std::string);
46 #define YY_USER_ACTION \
47 yylloc.first_column = yylloc.last_column; yylloc.last_column += yyleng;
48 //yylloc.last_column += yyleng;
50 /* -*- Verbose Special Debug -*- */
55 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
57 #define DEBUG(x) /* Nothing */
61 std::string token_to_string(int);
82 number [0-9]+[\.][0-9]*
86 floating_D ({little}|{number}|{integer})[dD][+-]?{integer}
87 floating_E ({little}|{number}|{integer})[eE][+-]?{integer}
93 utf2 ([\xC2-\xDF][\x80-\xBF])
94 utf31 ([\xE0][\xA0-\xBF][\x80-\xBF])
95 utf32 ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
96 utf33 ([\xED][\x80-\x9F][\x80-\xBF])
97 utf34 ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
98 utf41 ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
99 utf42 ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
100 utf43 ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
102 utf3 ({utf31}|{utf32}|{utf33}|{utf34})
103 utf4 ({utf41}|{utf42}|{utf43})
105 utf ({utf2}|{utf3}|{utf4})
106 id ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
109 newline ("\r"|"\n"|"\r\n")
110 blankline {spaces}+{newline}
111 emptyline {newline}({spaces}|[,;])+{newline}
113 char_in_line_comment [^\r\n]*
114 char_in_comment [^\r\n\/*]*
118 boolfalse ("%f"|"%F")
140 startblockcomment "/*"
145 in_string [^\"\'\r\n\.]*
152 dotpower (".^"|".**")
161 equal "="{spaces}*"="
162 notequal ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
165 lowerequal "<"{spaces}*"="
166 greaterequal ">"{spaces}*"="
172 controltimes ("*."[^0-9])
173 controlrdivide ("/."[^0-9])
174 controlldivide ("\\."[^0-9])
180 // BOM found ==> ignored
183 <INITIAL,BEGINID>{booltrue} {
185 return scan_throw(BOOLTRUE);
187 <INITIAL,BEGINID>{boolfalse} {
189 return scan_throw(BOOLFALSE);
192 <INITIAL,BEGINID>"if" {
193 if (last_token != DOT)
195 ParserSingleInstance::pushControlStatus(Parser::WithinIf);
197 DEBUG("BEGIN(INITIAL)");
199 return scan_throw(IF);
202 <INITIAL,BEGINID>"then" {
203 DEBUG("BEGIN(INITIAL)");
205 return scan_throw(THEN);
208 <INITIAL,BEGINID>"else" {
209 if (last_token != DOT)
211 // Pop to step out IF
212 ParserSingleInstance::popControlStatus();
213 ParserSingleInstance::pushControlStatus(Parser::WithinElse);
215 DEBUG("BEGIN(INITIAL)");
217 return scan_throw(ELSE);
220 <INITIAL,BEGINID>"elseif" {
221 if (last_token != DOT)
223 ParserSingleInstance::popControlStatus();
224 ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
226 DEBUG("BEGIN(INITIAL)");
228 return scan_throw(ELSEIF);
231 <INITIAL,BEGINID>"end" {
232 if (last_token != DOT)
234 ParserSingleInstance::popControlStatus();
236 DEBUG("BEGIN(INITIAL)");
238 return scan_throw(END);
241 <INITIAL,BEGINID>"select" {
242 if (last_token != DOT)
244 ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
246 DEBUG("BEGIN(INITIAL)");
248 return scan_throw(SELECT);
251 <INITIAL,BEGINID>"switch" {
252 if (last_token != DOT)
254 ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
256 DEBUG("BEGIN(INITIAL)");
258 return scan_throw(SWITCH);
261 <INITIAL,BEGINID>"otherwise" {
262 if (last_token != DOT)
264 ParserSingleInstance::popControlStatus();
265 ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
267 DEBUG("BEGIN(INITIAL)");
269 return scan_throw(OTHERWISE);
272 <INITIAL,BEGINID>"case" {
273 if (last_token != DOT)
275 ParserSingleInstance::popControlStatus();
276 ParserSingleInstance::pushControlStatus(Parser::WithinCase);
278 DEBUG("BEGIN(INITIAL)");
280 return scan_throw(CASE);
283 <INITIAL,BEGINID>"function" {
284 if (last_token != DOT)
286 ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
288 DEBUG("BEGIN(INITIAL)");
290 return scan_throw(FUNCTION);
293 <INITIAL,BEGINID>"endfunction" {
294 if (last_token != DOT)
296 ParserSingleInstance::popControlStatus();
298 DEBUG("BEGIN(INITIAL)");
300 return scan_throw(ENDFUNCTION);
303 <INITIAL,BEGINID>"for" {
304 if (last_token != DOT)
306 ParserSingleInstance::pushControlStatus(Parser::WithinFor);
309 return scan_throw(FOR);
312 <INITIAL,BEGINID>"while" {
313 if (last_token != DOT)
315 ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
318 return scan_throw(WHILE);
321 <INITIAL,BEGINID>"do" {
323 return scan_throw(DO);
326 <INITIAL,BEGINID>"break" {
328 return scan_throw(BREAK);
331 <INITIAL,BEGINID>"continue" {
333 return scan_throw(CONTINUE);
336 <INITIAL,BEGINID>"try" {
337 ParserSingleInstance::pushControlStatus(Parser::WithinTry);
339 return scan_throw(TRY);
342 <INITIAL,BEGINID>"catch" {
343 // Pop to step out TRY
344 ParserSingleInstance::popControlStatus();
345 ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
347 return scan_throw(CATCH);
350 <INITIAL,BEGINID>"return" {
352 return scan_throw(RETURN);
355 <INITIAL,BEGINID>"resume" {
357 return scan_throw(RETURN);
360 ^{spaces}*/({id}){spaces}([^ \t\v\f(=<>~@]|([~@]{spaces}*[^=]?)) {
367 wchar_t *pwText = to_wide_string(yytext);
368 if (yytext != NULL && pwText == NULL)
370 std::string str = "Can\'t convert \'";
372 str += "\' to UTF-8";
375 return scan_throw(FLEX_ERROR);
377 yylval.str = new std::wstring(pwText);
379 types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
380 if (pIT && pIT->isCallable())
388 return scan_throw(ID);
394 <INITIAL,MATRIX>{boolnot} {
395 return scan_throw(NOT);
397 <INITIAL,MATRIX>{dollar} {
398 return scan_throw(DOLLAR);
400 <INITIAL,MATRIX>{booltrue} {
401 return scan_throw(BOOLTRUE);
403 <INITIAL,MATRIX>{boolfalse} {
404 return scan_throw(BOOLFALSE);
406 <INITIAL,MATRIX>{booland} {
407 return scan_throw(AND);
409 <INITIAL,MATRIX>{boolandand} {
410 return scan_throw(ANDAND);
412 <INITIAL,MATRIX>{boolor} {
413 return scan_throw(OR);
415 <INITIAL,MATRIX>{booloror} {
416 return scan_throw(OROR);
421 return scan_throw(LPAREN);
424 return scan_throw(RPAREN);
428 <INITIAL,MATRIX>{semicolon} {
430 return scan_throw(SEMI);
433 <INITIAL,MATRIX>{comma} {
435 return scan_throw(COMMA);
438 <INITIAL,MATRIX>{colon} {
439 return scan_throw(COLON);
443 <INITIAL,MATRIX>{lbrace} {
444 yy_push_state(MATRIX);
445 paren_levels.push(0);
446 ParserSingleInstance::pushControlStatus(Parser::WithinCell);
447 return scan_throw(LBRACE);
451 return scan_throw(RBRACE);
455 <INITIAL,MATRIX>{dotquote} {
456 return scan_throw(DOTQUOTE);
458 <INITIAL,MATRIX>{dottimes} {
459 return scan_throw(DOTTIMES);
461 <INITIAL,MATRIX>{dotrdivide} {
462 return scan_throw(DOTRDIVIDE);
464 <INITIAL,MATRIX>{dotldivide} {
465 return scan_throw(DOTLDIVIDE);
467 <INITIAL,MATRIX>{dotpower} {
468 return scan_throw(DOTPOWER);
473 return scan_throw(MINUS);
476 return scan_throw(PLUS);
478 <INITIAL,MATRIX>{times} {
479 return scan_throw(TIMES);
481 <INITIAL,MATRIX>{rdivide} {
482 return scan_throw(RDIVIDE);
484 <INITIAL,MATRIX>{ldivide} {
485 return scan_throw(LDIVIDE);
487 <INITIAL,MATRIX>{power} {
488 return scan_throw(POWER);
491 <INITIAL,MATRIX>{krontimes} {
492 return scan_throw(KRONTIMES);
494 <INITIAL,MATRIX>{kronrdivide} {
495 return scan_throw(KRONRDIVIDE);
497 <INITIAL,MATRIX>{kronldivide} {
498 return scan_throw(KRONLDIVIDE);
502 <INITIAL,MATRIX>{controltimes} {
503 unput(yytext[yyleng - 1]);
504 return scan_throw(CONTROLTIMES);
506 <INITIAL,MATRIX>{controlrdivide} {
507 unput(yytext[yyleng - 1]);
508 return scan_throw(CONTROLRDIVIDE);
510 <INITIAL,MATRIX>{controlldivide} {
511 unput(yytext[yyleng - 1]);
512 return scan_throw(CONTROLLDIVIDE);
516 <INITIAL,MATRIX>{equal} {
517 return scan_throw(EQ);
519 <INITIAL,MATRIX>{notequal} {
520 return scan_throw(NE);
522 <INITIAL,MATRIX>{lowerthan} {
523 return scan_throw(LT);
525 <INITIAL,MATRIX>{greaterthan} {
526 return scan_throw(GT);
528 <INITIAL,MATRIX>{lowerequal} {
529 return scan_throw(LE);
531 <INITIAL,MATRIX>{greaterequal} {
532 return scan_throw(GE);
536 <INITIAL,MATRIX>{assign} {
537 return scan_throw(ASSIGN);
541 <INITIAL,MATRIX>{lbrack} {
542 DEBUG("yy_push_state(MATRIX)");
543 yy_push_state(MATRIX);
544 paren_levels.push(0);
545 ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
546 return scan_throw(LBRACK);
550 return scan_throw(RBRACK);
554 <INITIAL,MATRIX>{dot} {
555 return scan_throw(DOT);
559 ParserSingleInstance::pushControlStatus(Parser::WithinDots);
560 yy_push_state(LINEBREAK);
563 <INITIAL,MATRIX>{integer} {
564 yylval.number = atof(yytext);
566 std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
569 return scan_throw(VARINT);
573 <INITIAL,MATRIX>{floating_D} {
574 scan_exponent_convert(yytext);
575 yylval.number = atof(yytext);
577 std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
580 return scan_throw(VARFLOAT);
583 <INITIAL,MATRIX>{floating_E} {
584 yylval.number = atof(yytext);
586 std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
589 return scan_throw(VARFLOAT);
592 <INITIAL,MATRIX>[0-9]+[\.]/[\*^\\\/] {
594 yylval.number = atof(yytext);
596 std::cout << "--> [DEBUG] NUMBER WITH DOT AS LAST CHARACTER : " << yytext << std::endl;
600 yylloc.last_column--;
601 return scan_throw(NUM);
604 <INITIAL,MATRIX>{number} {
605 yylval.number = atof(yytext);
607 std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
610 return scan_throw(NUM);
614 <INITIAL,MATRIX>{little} {
615 yylval.number = atof(yytext);
617 std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
620 return scan_throw(NUM);
624 <INITIAL,MATRIX>{id} {
625 wchar_t *pwText = to_wide_string(yytext);
626 if (yytext != NULL && pwText == NULL)
628 std::string str = "Can\'t convert \'";
630 str += "\' to UTF-8";
633 return scan_throw(FLEX_ERROR);
635 yylval.str = new std::wstring(pwText);
638 std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
641 return scan_throw(ID);
645 <INITIAL,MATRIX>{startblockcomment} {
646 yylval.comment = new std::wstring();
648 ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
649 yy_push_state(REGIONCOMMENT);
653 <INITIAL,MATRIX>{startcomment} {
655 yy_push_state(LINECOMMENT);
659 <INITIAL,MATRIX,SHELLMODE>{dquote} {
661 str_opener_column = yylloc.first_column;
662 yy_push_state(DOUBLESTRING);
666 <INITIAL,MATRIX,SHELLMODE>{quote} {
668 ** Matrix Transposition special behaviour
669 ** ID' []' toto()' are transposition call
672 || last_token == RBRACK
673 || last_token == RPAREN
674 || last_token == RBRACE
675 || last_token == VARINT
676 || last_token == VARFLOAT
678 || last_token == BOOLTRUE
679 || last_token == BOOLFALSE)
681 return scan_throw(QUOTE);
686 str_opener_column = yylloc.first_column;
687 yy_push_state(SIMPLESTRING);
692 <INITIAL,MATRIX>{spaces} {
698 yylloc.last_line += 1;
699 yylloc.last_column = 1;
701 if (last_token != EOL) {
702 return scan_throw(EOL);
708 <INITIAL,MATRIX>{blankline} {
709 yylloc.last_line += 1;
710 yylloc.last_column = 1;
712 if (last_token != EOL)
714 return scan_throw(EOL);
719 <INITIAL,MATRIX>{emptyline} {
720 yylloc.last_line += 2;
721 yylloc.last_column = 1;
723 if (last_token != EOL)
725 return scan_throw(EOL);
730 std::string str = "Unexpected token \'";
735 return scan_throw(FLEX_ERROR);
742 ++paren_levels.top();
743 return scan_throw(LPAREN);
747 --paren_levels.top();
748 return scan_throw(RPAREN);
752 unput(yytext[yyleng -1]);
754 || last_token == RPAREN
755 || last_token == QUOTE
756 || last_token == VARINT
757 || last_token == VARFLOAT
758 || last_token == NUM)
760 return scan_throw(COMMA);
764 {spaces}*{colon}{spaces}* {
765 return scan_throw(COLON);
769 yylloc.last_line += 1;
770 yylloc.last_column = 1;
771 if(last_token != DOTS && last_token != EOL)
773 return scan_throw(EOL);
779 DEBUG("yy_pop_state()");
782 ParserSingleInstance::popControlStatus();
783 return scan_throw(RBRACK);
789 ParserSingleInstance::popControlStatus();
790 return scan_throw(RBRACE);
794 {spaces}{plus}{spaces} {
795 // _+_ is always meaning a PLUS token
796 // + alone is a plus if and only if it does not
797 // fall into the {spaces}{plus} rule (in matrices space is coding)
798 // as Flex is greedy the {plus} rule is only applied
799 // if no {spaces}{plus} is found
802 // [1 + 2 3] must be understood as [(1+2), 3]
803 // [1 +2 3] must be understood as [1, 2, 3]
805 // 2] must be understood as [(1+2)]
806 return scan_throw(PLUS);
810 {spaces}{minus}{spaces} {
811 // _-_ is always meaning a MINUS token
812 // - alone is a MINUS only if and only if it does not
813 // fall into the {spaces}{minus} rule (in matrices space is coding)
814 // as Flex is greedy the {minus} rule is only applied
815 // if no {spaces}{minus} is found
818 // [1 - 2 3] must be understood as [(1-2), 3]
819 // [1 -2 3] must be understood as [1, -2, 3]
821 // 2] must be understood as [(1-2)]
822 return scan_throw(MINUS);
826 // This rule is made to take into account the coding spaces in matrices.
827 // It is important to distinguish between a space coding a matrix column separator
828 // and a simple non coding space around the PLUS operator
831 // [a + b] == [(a + b)]
832 // but [a +b] == [a, b] and plus here is unary
833 // the space is non coding:
834 // * after any other binary operator __op__
835 // Example : [a __op__ +b]
836 // * after brackets or parentheses delimiters
837 // Example : [(1*2*a) +3]
838 // * at the beginning of a line
842 // no need to unput the '+'
843 if (last_token != LBRACK
845 && last_token != SEMI
846 && last_token != COMMA
847 && last_token != DOTTIMES
848 && last_token != DOTRDIVIDE
849 && last_token != DOTLDIVIDE
850 && last_token != DOTPOWER
851 && last_token != MINUS
852 && last_token != PLUS
853 && last_token != TIMES
854 && last_token != RDIVIDE
855 && last_token != LDIVIDE
856 && last_token != POWER
857 && last_token != KRONTIMES
858 && last_token != KRONRDIVIDE
859 && last_token != KRONLDIVIDE
866 && paren_levels.top() == 0)
868 return scan_throw(COMMA);
873 yylloc.last_column--;
878 // See {spaces}{plus} rule for the rationale
881 yylloc.last_column--;
882 if (last_token != LBRACK
884 && last_token != SEMI
885 && last_token != COMMA
886 && last_token != DOTTIMES
887 && last_token != DOTRDIVIDE
888 && last_token != DOTLDIVIDE
889 && last_token != DOTPOWER
890 && last_token != MINUS
891 && last_token != PLUS
892 && last_token != TIMES
893 && last_token != RDIVIDE
894 && last_token != LDIVIDE
895 && last_token != POWER
896 && last_token != KRONTIMES
897 && last_token != KRONRDIVIDE
898 && last_token != KRONLDIVIDE
905 && paren_levels.top() == 0)
907 return scan_throw(COMMA);
912 std::string str = "Unexpected token \'";
914 str += "\' within a matrix.";
917 return scan_throw(FLEX_ERROR);
922 * Scilab can perform a line continuation with the ..
923 * In matrices as space may be coding extra care must be taken when parsing {next}
924 * Some states must be preserved to parse next line and to revert to a proper state
925 * after the ... // comments or ... \/* comments *\/
928 {spaces}*{plus}{next} {
929 // This rule is made to take into account a +... without spaces after plus
930 // if one simply ignores the next a situation like this could arise
936 // what is meant by the user [1 +2] ? or [1 + 2]
937 // simply ignoring the ... would yield the 1st situation [1, 2]
938 // We consider this is NOT proper and instead that the user meant a binary plus
939 // split is two lines
940 // The same rationale applies to minus.
942 linebreak_stored_space = FALSE; // no spaces before ...
943 linebreak_stored_token = PLUS; // keep last token to restore
944 ParserSingleInstance::pushControlStatus(Parser::WithinDots);
945 yy_push_state(LINEBREAK);
946 return scan_throw(PLUS);
949 {spaces}*{minus}{next} {
950 // see {spaces}*{minus}{next} for the rationale
952 linebreak_stored_space = FALSE; // no spaces before ...
953 linebreak_stored_token = MINUS; // keep last token to restore
954 ParserSingleInstance::pushControlStatus(Parser::WithinDots);
955 yy_push_state(LINEBREAK);
956 return scan_throw(MINUS);
960 // Store the state of the previously scanned token for next rule
961 // Only considerations of coding spaces is important for the parser
963 linebreak_stored_space = FALSE; // no spaces before ...
964 linebreak_stored_token = last_token; // keep last token to restore state
965 ParserSingleInstance::pushControlStatus(Parser::WithinDots);
966 yy_push_state(LINEBREAK);
970 // Store the state of the previously scanned token for next rule
971 // Only considerations of coding spaces is important for the parser
973 linebreak_stored_space = TRUE; // no spaces before ...
974 linebreak_stored_token = last_token; // keep last token to restore state
975 ParserSingleInstance::pushControlStatus(Parser::WithinDots);
976 yy_push_state(LINEBREAK);
988 yylloc.last_line += 1;
989 yylloc.last_column = 1;
991 last_token = linebreak_stored_token;
992 if (linebreak_stored_space)
994 // This is important to restore coding spaces as if ... was not present
996 linebreak_stored_space = FALSE;
999 ParserSingleInstance::popControlStatus();
1002 {startblockcomment} {
1003 yylval.comment = new std::wstring();
1005 ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
1006 yy_push_state(REGIONCOMMENT);
1012 yy_push_state(LINECOMMENT);
1023 // The following case is not handled by the parser
1024 // a line of code ... /* some multiline
1025 // comments */ continued here;
1026 // without the special case telling we are after comments
1027 // will generate the error as follows:
1029 // Any characters after ... yields to an error
1030 std::string str = "Unexpected token \'";
1032 str += "\' after line break with .. or ...";
1034 ParserSingleInstance::popControlStatus();
1037 return scan_throw(FLEX_ERROR);
1045 //yylloc.last_line += 1;
1046 //yylloc.last_column = 1;
1049 for (int i = yyleng - 1 ; i >= 0 ; --i)
1051 //std::cerr << "Unputting i = {" << i << "}" << std::endl;
1052 //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
1054 yylloc.last_column--;
1057 ** To forgot comments after lines break
1059 if (last_token != DOTS)
1061 //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
1062 //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
1063 wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1064 //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
1065 if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1068 std::string str = "Can\'t convert \'";
1069 str += pstBuffer.c_str();
1070 str += "\' to UTF-8";
1073 return scan_throw(FLEX_ERROR);
1075 yylval.comment = new std::wstring(pwstBuffer);
1078 return scan_throw(COMMENT);
1088 wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1089 if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1092 std::string str = "Can\'t convert \'";
1093 str += pstBuffer.c_str();
1094 str += "\' to UTF-8";
1097 return scan_throw(FLEX_ERROR);
1099 yylval.comment = new std::wstring(pwstBuffer);
1102 return scan_throw(COMMENT);
1105 {char_in_line_comment} {
1106 // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
1107 // only translate to WCHAR_T when popping state.
1108 pstBuffer += yytext;
1117 if (comment_level == 0) {
1118 ParserSingleInstance::popControlStatus();
1120 //return scan_throw(BLOCKCOMMENT);
1124 {startblockcomment} {
1126 yy_push_state(REGIONCOMMENT);
1130 yylloc.last_line += 1;
1131 yylloc.last_column = 1;
1133 *yylval.comment += L"\n//";
1138 wchar_t *pwText = to_wide_string(yytext);
1139 *yylval.comment += std::wstring(pwText);
1145 // std::string str = "unexpected end of file in a comment";
1172 wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1173 if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1176 std::string str = "Can\'t convert \'";
1177 str += pstBuffer.c_str();
1178 str += "\' to UTF-8";
1181 return scan_throw(FLEX_ERROR);
1183 yylval.str = new std::wstring(pwstBuffer);
1186 yylloc.first_column = str_opener_column;
1187 return scan_throw(STR);
1193 yyerror("Heterogeneous string detected, starting with \' and ending with \".");
1194 return scan_throw(FLEX_ERROR);
1198 yylloc.last_line += 1;
1199 yylloc.last_column = 1;
1200 /* Do nothing... Just skip */
1205 yylloc.last_line += 1;
1206 yylloc.last_column = 1;
1208 yyerror("Unexpected end of line in a string.");
1209 return scan_throw(FLEX_ERROR);
1215 yyerror("Unexpected end of file in a string.");
1216 return scan_throw(FLEX_ERROR);
1222 pstBuffer += yytext;
1248 wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1249 if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1252 std::string str = "Can\'t convert \'";
1253 str += pstBuffer.c_str();
1254 str += "\' to UTF-8";
1257 return scan_throw(FLEX_ERROR);
1259 yylval.str = new std::wstring(pwstBuffer);
1262 yylloc.first_column = str_opener_column;
1263 return scan_throw(STR);
1269 yyerror("Heterogeneous string detected, starting with \" and ending with \'.");
1270 return scan_throw(FLEX_ERROR);
1274 yylloc.last_line += 1;
1275 yylloc.last_column = 1;
1276 /* Do nothing... Just skip */
1281 yylloc.last_line += 1;
1282 yylloc.last_column = 1;
1284 yyerror("Unexpected end of line in a string.");
1285 return scan_throw(FLEX_ERROR);
1291 yyerror("Unexpected end of file in a string.");
1292 return scan_throw(FLEX_ERROR);
1298 pstBuffer += yytext;
1306 if (last_token == ID)
1316 return scan_throw(SEMI);
1322 return scan_throw(COMMA);
1327 yylloc.last_line += 1;
1328 yylloc.last_column = 1;
1330 return scan_throw(EOL);
1334 if (last_token == STR || last_token == SPACES)
1336 wchar_t *pwText = to_wide_string(yytext);
1337 yylval.str = new std::wstring(pwText);
1339 return scan_throw(STR);
1344 return scan_throw(ASSIGN);
1349 if (last_token == STR || last_token == SPACES)
1351 wchar_t *pwText = to_wide_string(yytext);
1352 yylval.str = new std::wstring(pwText);
1354 return scan_throw(STR);
1359 return scan_throw(LPAREN);
1364 if (last_token == STR || last_token == SPACES)
1366 wchar_t *pwText = to_wide_string(yytext);
1367 yylval.str = new std::wstring(pwText);
1369 return scan_throw(STR);
1374 return scan_throw(LT);
1379 if (last_token == STR || last_token == SPACES)
1381 wchar_t *pwText = to_wide_string(yytext);
1382 yylval.str = new std::wstring(pwText);
1384 return scan_throw(STR);
1389 return scan_throw(GT);
1394 if (last_token == STR || last_token == SPACES)
1396 wchar_t *pwText = to_wide_string(yytext);
1397 yylval.str = new std::wstring(pwText);
1399 return scan_throw(STR);
1404 return scan_throw(NOT);
1409 [^ \t\v\f\r\n,;'"]+ {
1410 wchar_t *pwText = to_wide_string(yytext);
1411 yylval.str = new std::wstring(pwText);
1413 return scan_throw(STR);
1424 int scan_throw(int token) {
1427 std::cout << "--> [DEBUG] TOKEN : " << token << " - " << token_to_string(token) << std::endl;
1432 int get_last_token() {
1437 yylloc.first_line = yylloc.last_line;
1438 yylloc.first_column = yylloc.last_column;
1442 ** convert floating numbers to C standard
1446 void scan_exponent_convert(char *in)
1448 for (; *in != 'd' && *in != 'D'; ++in);
1453 int isatty (int desc)
1460 std::string token_to_string(int token)
1465 case AND : str = "AND";
1467 case ASSIGN : str = "ASSIGN";
1469 case BOOLFALSE : str = "BOOLFALSE";
1471 case BOOLTRUE : str = "BOOLTRUE";
1473 case BREAK : str = "BREAK";
1475 case CASE : str = "CASE";
1477 case CATCH : str = "CATCH";
1479 case COLON : str = "COLON";
1481 case COMMA : str = "COMMA";
1483 case COMMENT : str = "COMMENT";
1485 case CONTINUE : str = "CONTINUE";
1487 case CONTROLLDIVIDE : str = "CONTROLLDIVIDE";
1489 case CONTROLRDIVIDE : str = "CONTROLRDIVIDE";
1491 case CONTROLTIMES : str = "CONTROLTIMES";
1493 case DO : str = "DO";
1495 case DOLLAR : str = "DOLLAR";
1497 case DOT : str = "DOT";
1499 case DOTLDIVIDE : str = "DOTLDIVIDE";
1501 case DOTPOWER : str = "DOTPOWER";
1503 case DOTQUOTE : str = "DOTQUOTE";
1505 case DOTRDIVIDE : str = "DOTRDIVIDE";
1507 case DOTS : str = "DOTS";
1509 case DOTTIMES : str = "DOTTIMES";
1511 case ELSE : str = "ELSE";
1513 case ELSEIF : str = "ELSEIF";
1515 case END : str = "END";
1517 case ENDFUNCTION : str = "ENDFUNCTION";
1519 case EOL : str = "EOL";
1521 case EQ : str = "EQ";
1523 case FLEX_ERROR : str = "FLEX_ERROR";
1525 case FOR : str = "FOR";
1527 case FUNCTION : str = "FUNCTION";
1529 case GE : str = "GE";
1531 case GT : str = "GT";
1533 case ID : str = "ID";
1535 case IF : str = "IF";
1537 case KRONLDIVIDE : str = "KRONLDIVIDE";
1539 case KRONRDIVIDE : str = "KRONRDIVIDE";
1541 case KRONTIMES : str = "KRONTIMES";
1543 case LBRACE : str = "LBRACE";
1545 case LBRACK : str = "LBRACK";
1547 case LDIVIDE : str = "LDIVIDE";
1549 case LE : str = "LE";
1551 case LPAREN : str = "LPAREN";
1553 case LT : str = "LT";
1555 case MINUS : str = "MINUS";
1557 case NE : str = "NE";
1559 case NOT : str = "NOT";
1561 case NUM : str = "NUM";
1563 case OR : str = "OR";
1565 case OROR : str = "OROR";
1567 case OTHERWISE : str = "OTHERWISE";
1569 case PLUS : str = "PLUS";
1571 case POWER : str = "POWER";
1573 case QUOTE : str = "QUOTE";
1575 case RBRACE : str = "RBRACE";
1577 case RBRACK : str = "RBRACK";
1579 case RDIVIDE : str = "RDIVIDE";
1581 case RETURN : str = "RETURN";
1583 case RPAREN : str = "RPAREN";
1585 case SELECT : str = "SELECT";
1587 case SEMI : str = "SEMI";
1589 case SPACES : str = "SPACES";
1591 case STR : str = "STR";
1593 case SWITCH : str = "SWITCH";
1595 case THEN : str = "THEN";
1597 case TIMES : str = "TIMES";
1599 case TRY : str = "TRY";
1601 case VARFLOAT : str = "VARFLOAT";
1603 case VARINT : str = "VARINT";
1605 case WHILE : str = "WHILE";
1607 default : str = "UNKNOWN";