Merge remote-tracking branch 'origin/master' into windows
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{
2 /* -*- C++ -*- */
3 /*
4  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
5  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
6  *
7  * Copyright (C) 2012 - 2016 - Scilab Enterprises
8  *
9  * This file is hereby licensed under the terms of the GNU GPL v2.0,
10  * pursuant to article 5.3.4 of the CeCILL v.2.1.
11  * This file was originally licensed under the terms of the CeCILL v2.1,
12  * and continues to be available under such terms.
13  * For more information, see the COPYING file which you should have received
14  * along with this program.
15  *
16  */
17
18 #include <stack>
19
20 #include "isatty.hxx"
21 #include "parse.hxx"
22 #include "parser_private.hxx"
23
24 #include "context.hxx"
25
26 extern "C"
27 {
28 #include "charEncoding.h"
29 #include "sci_malloc.h"
30 }
31
32 static std::stack<int> paren_levels;
33
34 static int comment_level = 0;
35 static int last_token = 0;
36 static int linebreak_stored_token = 0;
37 static bool linebreak_stored_space = FALSE;
38 static int exit_status = PARSE_ERROR;
39 static int str_opener_column = 0;
40 static std::string current_file;
41 static std::string program_name;
42
43 static std::string pstBuffer;
44
45 extern void yyerror(std::string);
46
47 #define YY_USER_ACTION                          \
48     yylloc.first_column = yylloc.last_column; yylloc.last_column += yyleng;
49 //yylloc.last_column += yyleng;
50
51 /* -*- Verbose Special Debug -*- */
52 //#define DEV
53 //#define TOKENDEV
54
55 #ifdef DEV
56 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
57 #else
58 #define DEBUG(x) /* Nothing */
59 #endif
60
61 #ifdef DEV
62 std::string token_to_string(int);
63 #endif
64
65 %}
66
67 %option stack
68 %option noyywrap
69
70 %x SIMPLESTRING
71 %x DOUBLESTRING
72 %x REGIONCOMMENT
73 %x LINECOMMENT
74 %x LINEBREAK
75
76 %x MATRIX
77
78 %x SHELLMODE
79 %x BEGINID
80
81 spaces                  [ \t\v\f]+
82 integer                 [0-9]+
83 number                  [0-9]+[\.][0-9]*
84 little                  \.[0-9]+
85 bom             \xEF\xBB\xBF
86
87 floating_D              ({little}|{number}|{integer})[dD][+-]?{integer}
88 floating_E              ({little}|{number}|{integer})[eE][+-]?{integer}
89
90 hex             [0]x[0-9a-fA-F]+
91 oct             [0]o[0-7]+
92
93
94 utf2            ([\xC2-\xDF][\x80-\xBF])
95 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
96 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
97 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
98 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
99 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
100 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
101 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
102
103 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
104 utf4            ({utf41}|{utf42}|{utf43})
105
106 utf             ({utf2}|{utf3}|{utf4})
107 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
108
109
110 newline                 ("\r"|"\n"|"\r\n")
111 blankline               {spaces}+{newline}
112 emptyline       {newline}({spaces}|[,;])+{newline}
113 next                    \.\.+
114 char_in_line_comment    [^\r\n]*
115 char_in_comment         [^\r\n\/*]*
116
117 boolnot                 ("@"|"~")
118 booltrue                ("%t"|"%T")
119 boolfalse               ("%f"|"%F")
120 booland                 ("&")
121 boolandand              ("&&")
122 boolor                  ("|")
123 booloror                ("||")
124
125 lbrack                  "["
126 rbrack                  "]"
127
128 lparen                  "("
129 rparen                  ")"
130
131 lbrace                  "{"
132 rbrace                  "}"
133
134 dollar                  "$"
135
136 semicolon               ";"
137 comma                   ","
138 colon                   ":"
139
140 startcomment            "//"
141 startblockcomment       "/*"
142 endblockcomment         "*/"
143
144 dquote                  "\""
145 quote                   "'"
146 in_string               [^\"\'\r\n\.]*
147
148 dot             "."
149 dotquote                ".'"
150 dottimes                ".*"
151 dotrdivide              "./"
152 dotldivide              ".\\"
153 dotpower                (".^"|".**")
154
155 plus                    "+"
156 minus                   "-"
157 rdivide                 "/"
158 ldivide                 "\\"
159 times                   "*"
160 power                   ("^"|"**")
161
162 equal                   "="{spaces}*"="
163 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
164 lowerthan               "<"
165 greaterthan             ">"
166 lowerequal              "<"{spaces}*"="
167 greaterequal            ">"{spaces}*"="
168
169 krontimes               ".*."
170 kronrdivide             "./."
171 kronldivide             ".\\."
172
173 controltimes    ("*."[^0-9])
174 controlrdivide  ("/."[^0-9])
175 controlldivide  ("\\."[^0-9])
176
177 assign                  "="
178
179 %%
180 <INITIAL>{bom}/.* {
181 // BOM found ==> ignored
182 }
183
184 <INITIAL,BEGINID>{booltrue}     {
185     BEGIN(INITIAL);
186     return scan_throw(BOOLTRUE);
187 }
188 <INITIAL,BEGINID>{boolfalse}    {
189     BEGIN(INITIAL);
190     return scan_throw(BOOLFALSE);
191 }
192
193 <INITIAL,BEGINID>"if"            {
194         if (last_token != DOT)
195     {
196         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
197     }
198     DEBUG("BEGIN(INITIAL)");
199     BEGIN(INITIAL);
200     return scan_throw(IF);
201 }
202
203 <INITIAL,BEGINID>"then"          {
204     DEBUG("BEGIN(INITIAL)");
205     BEGIN(INITIAL);
206     return scan_throw(THEN);
207 }
208
209 <INITIAL,BEGINID>"else"          {
210         if (last_token != DOT)
211     {
212         // Pop to step out IF
213         ParserSingleInstance::popControlStatus();
214         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
215     }
216     DEBUG("BEGIN(INITIAL)");
217     BEGIN(INITIAL);
218         return scan_throw(ELSE);
219 }
220
221 <INITIAL,BEGINID>"elseif" {
222         if (last_token != DOT)
223     {
224         ParserSingleInstance::popControlStatus();
225         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
226     }
227     DEBUG("BEGIN(INITIAL)");
228     BEGIN(INITIAL);
229         return scan_throw(ELSEIF);
230 }
231
232 <INITIAL,BEGINID>"end"          {
233         if (last_token != DOT)
234     {
235         ParserSingleInstance::popControlStatus();
236     }
237     DEBUG("BEGIN(INITIAL)");
238     BEGIN(INITIAL);
239     return scan_throw(END);
240 }
241
242 <INITIAL,BEGINID>"select"       {
243         if (last_token != DOT)
244     {
245         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
246     }
247     DEBUG("BEGIN(INITIAL)");
248     BEGIN(INITIAL);
249     return scan_throw(SELECT);
250 }
251
252 <INITIAL,BEGINID>"switch"       {
253         if (last_token != DOT)
254     {
255         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
256     }
257     DEBUG("BEGIN(INITIAL)");
258     BEGIN(INITIAL);
259     return scan_throw(SWITCH);
260 }
261
262 <INITIAL,BEGINID>"otherwise" {
263         if (last_token != DOT)
264     {
265         ParserSingleInstance::popControlStatus();
266         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
267     }
268     DEBUG("BEGIN(INITIAL)");
269     BEGIN(INITIAL);
270         return scan_throw(OTHERWISE);
271 }
272
273 <INITIAL,BEGINID>"case"         {
274         if (last_token != DOT)
275     {
276         ParserSingleInstance::popControlStatus();
277         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
278     }
279     DEBUG("BEGIN(INITIAL)");
280     BEGIN(INITIAL);
281     return scan_throw(CASE);
282 }
283
284 <INITIAL,BEGINID>"function" {
285         if (last_token != DOT)
286     {
287         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
288     }
289     DEBUG("BEGIN(INITIAL)");
290     BEGIN(INITIAL);
291     return scan_throw(FUNCTION);
292 }
293
294 <INITIAL,BEGINID>"endfunction" {
295         if (last_token != DOT)
296     {
297         ParserSingleInstance::popControlStatus();
298     }
299     DEBUG("BEGIN(INITIAL)");
300     BEGIN(INITIAL);
301         return scan_throw(ENDFUNCTION);
302 }
303
304 <INITIAL,BEGINID>"for" {
305         if (last_token != DOT)
306     {
307         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
308     }
309     BEGIN(INITIAL);
310     return scan_throw(FOR);
311 }
312
313 <INITIAL,BEGINID>"while"        {
314         if (last_token != DOT)
315     {
316         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
317     }
318         BEGIN(INITIAL);
319         return scan_throw(WHILE);
320 }
321
322 <INITIAL,BEGINID>"do"           {
323         BEGIN(INITIAL);
324     return scan_throw(DO);
325 }
326
327 <INITIAL,BEGINID>"break"                {
328         BEGIN(INITIAL);
329         return scan_throw(BREAK);
330 }
331
332 <INITIAL,BEGINID>"continue"             {
333         BEGIN(INITIAL);
334         return scan_throw(CONTINUE);
335 }
336
337 <INITIAL,BEGINID>"try" {
338         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
339         BEGIN(INITIAL);
340         return scan_throw(TRY);
341 }
342
343 <INITIAL,BEGINID>"catch" {
344     // Pop to step out TRY
345         ParserSingleInstance::popControlStatus();
346         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
347         BEGIN(INITIAL);
348         return scan_throw(CATCH);
349 }
350
351 <INITIAL,BEGINID>"return"       {
352     BEGIN(INITIAL);
353     return scan_throw(RETURN);
354 }
355
356 <INITIAL,BEGINID>"resume"       {
357     BEGIN(INITIAL);
358     return scan_throw(RETURN);
359 }
360
361 ^{spaces}*/({id}){spaces}([^ \t\v\f(=<>~@,;]|([~@]{spaces}*[^=]?)) {
362         BEGIN(BEGINID);
363 }
364
365 <BEGINID>
366 {
367     {id}                        {
368         yylval.str = new std::string(yytext);
369         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
370         if (pIT && pIT->isCallable())
371         {
372             BEGIN(SHELLMODE);
373         }
374         else
375         {
376             BEGIN(INITIAL);
377         }
378         return scan_throw(ID);
379     }
380
381 }
382
383 <INITIAL,MATRIX>{boolnot}               {
384   return scan_throw(NOT);
385 }
386 <INITIAL,MATRIX>{dollar}                {
387   return scan_throw(DOLLAR);
388 }
389 <INITIAL,MATRIX>{booltrue}              {
390   return scan_throw(BOOLTRUE);
391 }
392 <INITIAL,MATRIX>{boolfalse}             {
393   return scan_throw(BOOLFALSE);
394 }
395 <INITIAL,MATRIX>{booland}               {
396   return scan_throw(AND);
397 }
398 <INITIAL,MATRIX>{boolandand}    {
399   return scan_throw(ANDAND);
400 }
401 <INITIAL,MATRIX>{boolor}                {
402   return scan_throw(OR);
403 }
404 <INITIAL,MATRIX>{booloror}              {
405   return scan_throw(OROR);
406 }
407
408
409 <INITIAL>{lparen}                       {
410   return scan_throw(LPAREN);
411 }
412 <INITIAL>{rparen}                       {
413   return scan_throw(RPAREN);
414 }
415
416
417 <INITIAL,MATRIX>{semicolon}             {
418         scan_step();
419   return scan_throw(SEMI);
420 }
421
422 <INITIAL,MATRIX>{comma}                 {
423         scan_step();
424   return scan_throw(COMMA);
425 }
426
427 <INITIAL,MATRIX>{colon}                 {
428   return scan_throw(COLON);
429 }
430
431
432 <INITIAL,MATRIX>{lbrace}                {
433   yy_push_state(MATRIX);
434   paren_levels.push(0);
435   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
436   return scan_throw(LBRACE);
437 }
438
439 {rbrace}                        {
440   return scan_throw(RBRACE);
441 }
442
443
444 <INITIAL,MATRIX>{dotquote}              {
445   return scan_throw(DOTQUOTE);
446 }
447 <INITIAL,MATRIX>{dottimes}              {
448   return scan_throw(DOTTIMES);
449 }
450 <INITIAL,MATRIX>{dotrdivide}            {
451   return scan_throw(DOTRDIVIDE);
452 }
453 <INITIAL,MATRIX>{dotldivide}            {
454   return scan_throw(DOTLDIVIDE);
455 }
456 <INITIAL,MATRIX>{dotpower}              {
457   return scan_throw(DOTPOWER);
458 }
459
460
461 {minus}                                 {
462   return scan_throw(MINUS);
463 }
464 {plus}                                  {
465   return scan_throw(PLUS);
466 }
467 <INITIAL,MATRIX>{times}                 {
468   return scan_throw(TIMES);
469 }
470 <INITIAL,MATRIX>{rdivide}               {
471   return scan_throw(RDIVIDE);
472 }
473 <INITIAL,MATRIX>{ldivide}               {
474   return scan_throw(LDIVIDE);
475 }
476 <INITIAL,MATRIX>{power}                 {
477   return scan_throw(POWER);
478 }
479
480 <INITIAL,MATRIX>{krontimes}             {
481   return scan_throw(KRONTIMES);
482 }
483 <INITIAL,MATRIX>{kronrdivide}           {
484   return scan_throw(KRONRDIVIDE);
485 }
486 <INITIAL,MATRIX>{kronldivide}           {
487   return scan_throw(KRONLDIVIDE);
488 }
489
490
491 <INITIAL,MATRIX>{controltimes}          {
492     unput(yytext[yyleng - 1]);
493     return scan_throw(CONTROLTIMES);
494 }
495 <INITIAL,MATRIX>{controlrdivide}                {
496     unput(yytext[yyleng - 1]);
497     return scan_throw(CONTROLRDIVIDE);
498 }
499 <INITIAL,MATRIX>{controlldivide}                {
500     unput(yytext[yyleng - 1]);
501     return scan_throw(CONTROLLDIVIDE);
502 }
503
504
505 <INITIAL,MATRIX>{equal}                 {
506   return scan_throw(EQ);
507 }
508 <INITIAL,MATRIX>{notequal}              {
509   return scan_throw(NE);
510 }
511 <INITIAL,MATRIX>{lowerthan}             {
512   return scan_throw(LT);
513 }
514 <INITIAL,MATRIX>{greaterthan}           {
515   return scan_throw(GT);
516 }
517 <INITIAL,MATRIX>{lowerequal}            {
518   return scan_throw(LE);
519 }
520 <INITIAL,MATRIX>{greaterequal}          {
521   return scan_throw(GE);
522 }
523
524
525 <INITIAL,MATRIX>{assign}                {
526   return scan_throw(ASSIGN);
527  }
528
529
530 <INITIAL,MATRIX>{lbrack}                {
531   DEBUG("yy_push_state(MATRIX)");
532   yy_push_state(MATRIX);
533   paren_levels.push(0);
534   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
535   return scan_throw(LBRACK);
536 }
537
538 <INITIAL>{rbrack}                               {
539   return scan_throw(RBRACK);
540 }
541
542
543 <INITIAL,MATRIX>{dot}                   {
544   return scan_throw(DOT);
545 }
546
547 <INITIAL>{next}                 {
548     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
549     yy_push_state(LINEBREAK);
550 }
551
552 <INITIAL,MATRIX>{integer}               {
553   yylval.number = atof(yytext);
554 #ifdef TOKENDEV
555   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
556 #endif
557 //  scan_step();
558   return scan_throw(VARINT);
559 }
560
561
562 <INITIAL,MATRIX>{floating_D}            {
563   scan_exponent_convert(yytext);
564   yylval.number = atof(yytext);
565 #ifdef TOKENDEV
566   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
567 #endif
568   //scan_step();
569   return scan_throw(VARFLOAT);
570 }
571
572 <INITIAL,MATRIX>{floating_E}            {
573   yylval.number = atof(yytext);
574 #ifdef TOKENDEV
575   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
576 #endif
577   //scan_step();
578   return scan_throw(VARFLOAT);
579 }
580
581 <INITIAL,MATRIX>[0-9]+[\.]/[\*^\\\/]            {
582
583   yylval.number = atof(yytext);
584 #ifdef TOKENDEV
585   std::cout << "--> [DEBUG] NUMBER WITH DOT AS LAST CHARACTER : " << yytext << std::endl;
586 #endif
587 //  scan_step();
588   unput('.');
589   yylloc.last_column--;
590   return scan_throw(NUM);
591 }
592
593 <INITIAL,MATRIX>{number}                {
594   yylval.number = atof(yytext);
595 #ifdef TOKENDEV
596   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
597 #endif
598 //  scan_step();
599   return scan_throw(NUM);
600 }
601
602
603 <INITIAL,MATRIX>{little}                {
604   yylval.number = atof(yytext);
605 #ifdef TOKENDEV
606   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
607 #endif
608 //  scan_step();
609   return scan_throw(NUM);
610 }
611
612
613 <INITIAL,MATRIX>{id}                    {
614     yylval.str = new std::string(yytext);
615 #ifdef TOKENDEV
616   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
617 #endif
618 //  scan_step();
619   return scan_throw(ID);
620 }
621
622
623 <INITIAL,MATRIX,SHELLMODE,BEGINID>{startblockcomment}   {
624   yylval.comment = new std::string();
625   comment_level = 1;
626   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
627   yy_push_state(REGIONCOMMENT);
628 }
629
630
631 <INITIAL,MATRIX,SHELLMODE,BEGINID>{startcomment}                {
632   pstBuffer.clear();
633   yy_push_state(LINECOMMENT);
634 }
635
636
637 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
638   pstBuffer.clear();
639   str_opener_column = yylloc.first_column;
640   yy_push_state(DOUBLESTRING);
641 }
642
643
644 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
645   /*
646   ** Matrix Transposition special behaviour
647   ** ID' []' toto()' are transposition call
648   */
649   if (last_token == ID
650       || last_token == RBRACK
651       || last_token == RPAREN
652       || last_token == RBRACE
653       || last_token == VARINT
654       || last_token == VARFLOAT
655       || last_token == NUM
656       || last_token == BOOLTRUE
657       || last_token == BOOLFALSE)
658   {
659       return scan_throw(QUOTE);
660   }
661   else
662   {
663       pstBuffer.clear();
664       str_opener_column = yylloc.first_column;
665       yy_push_state(SIMPLESTRING);
666   }
667 }
668
669
670 <INITIAL,MATRIX>{spaces}                {
671         scan_step();
672 }
673
674
675 <INITIAL>{newline}              {
676   yylloc.last_line += 1;
677   yylloc.last_column = 1;
678   scan_step();
679   if (last_token != EOL) {
680       return scan_throw(EOL);
681   }
682
683 }
684
685
686 <INITIAL,MATRIX>{blankline}             {
687   yylloc.last_line += 1;
688   yylloc.last_column = 1;
689   scan_step();
690   if (last_token != EOL)
691   {
692       return scan_throw(EOL);
693   }
694   scan_throw(EOL);
695 }
696
697 <INITIAL,MATRIX>{emptyline}             {
698   yylloc.last_line += 2;
699   yylloc.last_column = 1;
700   scan_step();
701   if (last_token != EOL)
702   {
703       return scan_throw(EOL);
704   }
705   scan_throw(EOL);
706 }
707 .                                       {
708     std::string str = "Unexpected token \'";
709     str += yytext;
710     str += "\'";
711     BEGIN(INITIAL);
712     yyerror(str);
713     return scan_throw(FLEX_ERROR);
714 }
715
716
717 <MATRIX>
718 {
719   {lparen} {
720     ++paren_levels.top();
721     return scan_throw(LPAREN);
722   }
723
724   {rparen} {
725     --paren_levels.top();
726     return scan_throw(RPAREN);
727   }
728
729   {spaces}+{lparen} {
730       unput(yytext[yyleng - 1]);
731       --yylloc.last_column;
732       if (last_token == ID
733           || last_token == RPAREN
734           || last_token == QUOTE
735           || last_token == VARINT
736           || last_token == VARFLOAT
737           || last_token == NUM)
738       {
739           return scan_throw(COMMA);
740       }
741   }
742
743   {spaces}*{colon}{spaces}* {
744       return scan_throw(COLON);
745   }
746
747   {newline} {
748       yylloc.last_line += 1;
749       yylloc.last_column = 1;
750       if(last_token != DOTS && last_token != EOL)
751       {
752           return scan_throw(EOL);
753       }
754       scan_throw(EOL);
755   }
756
757   {spaces}+{quote} {
758       unput('\'');
759       yylloc.last_column--;
760       if (last_token == ID
761           || last_token == RPAREN
762           || last_token == QUOTE)
763       {
764           return scan_throw(COMMA);
765       }
766   }
767
768   {rbrack}                              {
769     DEBUG("yy_pop_state()");
770     yy_pop_state();
771     paren_levels.pop();
772     ParserSingleInstance::popControlStatus();
773     return scan_throw(RBRACK);
774   }
775
776   {rbrace}                              {
777     yy_pop_state();
778     paren_levels.pop();
779     ParserSingleInstance::popControlStatus();
780     return scan_throw(RBRACE);
781   }
782
783   {plus}                                |
784   {spaces}{plus}{spaces}                {
785     // _+_ is always meaning a PLUS token
786     // + alone is a plus if and only if it does not
787     // fall into the {spaces}{plus} rule (in matrices space is coding)
788     // as Flex is greedy the {plus} rule is only applied
789     // if no {spaces}{plus} is found
790     // Examples:
791     // ========
792     // [1 + 2 3] must be understood as [(1+2), 3]
793     // [1 +2 3] must be understood as [1, 2, 3]
794     // [1 +...
795     // 2] must be understood as [(1+2)]
796     return scan_throw(PLUS);
797   }
798
799   {minus}                               |
800   {spaces}{minus}{spaces}               {
801     // _-_ is always meaning a MINUS token
802     // - alone is a MINUS only if and only if it does not
803     // fall into the {spaces}{minus} rule (in matrices space is coding)
804     // as Flex is greedy the {minus} rule is only applied
805     // if no {spaces}{minus} is found
806     // Examples:
807     // ========
808     // [1 - 2 3] must be understood as [(1-2), 3]
809     // [1 -2 3] must be understood as [1, -2, 3]
810     // [1 -...
811     // 2] must be understood as [(1-2)]
812     return scan_throw(MINUS);
813   }
814
815   {spaces}{plus}                        {
816     // This rule is made to take into account the coding spaces in matrices.
817     // It is important to distinguish between a space coding a matrix column separator
818     // and a simple non coding space around the PLUS operator
819     // Examples
820     // ========
821     // [a + b] == [(a + b)]
822     // but [a +b] == [a, b] and plus here is unary
823     // the space is non coding:
824     // * after any other binary operator __op__
825     //   Example : [a __op__ +b]
826     // * after brackets or parentheses delimiters
827     //   Example : [(1*2*a) +3]
828     // * at the beginning of a line
829     //   Example : [3 ...
830     //              _+2]
831
832     // no need to unput the '+'
833     if (last_token != LBRACK
834        && last_token != EOL
835        && last_token != SEMI
836        && last_token != COMMA
837        && last_token != DOTTIMES
838        && last_token != DOTRDIVIDE
839        && last_token != DOTLDIVIDE
840        && last_token != DOTPOWER
841        && last_token != MINUS
842        && last_token != PLUS
843        && last_token != TIMES
844        && last_token != RDIVIDE
845        && last_token != LDIVIDE
846        && last_token != POWER
847        && last_token != KRONTIMES
848        && last_token != KRONRDIVIDE
849        && last_token != KRONLDIVIDE
850        && last_token != EQ
851        && last_token != NE
852        && last_token != LT
853        && last_token != GT
854        && last_token != LE
855        && last_token != GE
856       && paren_levels.top() == 0)
857    {
858        return scan_throw(COMMA);
859    }
860    else
861    {
862        unput('+');
863        yylloc.last_column--;
864    }
865   }
866
867   {spaces}{minus}                       {
868     // See {spaces}{plus} rule for the rationale
869
870     unput('-');
871     yylloc.last_column--;
872     if (last_token != LBRACK
873        && last_token != EOL
874        && last_token != SEMI
875        && last_token != COMMA
876        && last_token != DOTTIMES
877        && last_token != DOTRDIVIDE
878        && last_token != DOTLDIVIDE
879        && last_token != DOTPOWER
880        && last_token != MINUS
881        && last_token != PLUS
882        && last_token != TIMES
883        && last_token != RDIVIDE
884        && last_token != LDIVIDE
885        && last_token != POWER
886        && last_token != KRONTIMES
887        && last_token != KRONRDIVIDE
888        && last_token != KRONLDIVIDE
889        && last_token != EQ
890        && last_token != NE
891        && last_token != LT
892        && last_token != GT
893        && last_token != LE
894        && last_token != GE
895        && paren_levels.top() == 0)
896    {
897        return scan_throw(COMMA);
898    }
899   }
900
901   .                                     {
902     std::string str = "Unexpected token \'";
903     str += yytext;
904     str += "\' within a matrix.";
905     BEGIN(INITIAL);
906     yyerror(str);
907     return scan_throw(FLEX_ERROR);
908   }
909
910   /* {next} rules
911    * ============
912    * Scilab can perform a line continuation with the ..
913    * In matrices as space may be coding extra care must be taken when parsing {next}
914    * Some states must be preserved to parse next line and to revert to a proper state
915    * after the ... // comments or ... \/* comments *\/
916    */
917
918   {spaces}*{plus}{next} {
919              // This rule is made to take into account a +... without spaces after plus
920              // if one simply ignores the next a situation like this could arise
921              // Example
922              // =======
923              // A = [1 +...
924              // 2] 
925              //
926              // what is meant by the user [1 +2] ? or [1 + 2]
927              // simply ignoring the ... would yield the 1st situation [1, 2]
928              // We consider this is NOT proper and instead that the user meant a binary plus
929              // split is two lines
930              // The same rationale applies to minus.
931
932              linebreak_stored_space = FALSE; // no spaces before ...
933              linebreak_stored_token = PLUS; // keep last token to restore
934              ParserSingleInstance::pushControlStatus(Parser::WithinDots);
935              yy_push_state(LINEBREAK);
936              return scan_throw(PLUS);
937          }
938
939   {spaces}*{minus}{next} {
940              // see {spaces}*{minus}{next} for the rationale
941
942              linebreak_stored_space = FALSE; // no spaces before ...
943              linebreak_stored_token = MINUS; // keep last token to restore
944              ParserSingleInstance::pushControlStatus(Parser::WithinDots);
945              yy_push_state(LINEBREAK);
946              return scan_throw(MINUS);
947          }
948
949   {next} {
950              // Store the state of the previously scanned token for next rule
951              // Only considerations of coding spaces is important for the parser
952
953              linebreak_stored_space = FALSE; // no spaces before ...
954              linebreak_stored_token = last_token; // keep last token to restore state
955              ParserSingleInstance::pushControlStatus(Parser::WithinDots);
956              yy_push_state(LINEBREAK);
957          }
958
959   {spaces}{next} {
960              // Store the state of the previously scanned token for next rule
961              // Only considerations of coding spaces is important for the parser
962
963              linebreak_stored_space = TRUE; // no spaces before ...
964              linebreak_stored_token = last_token; // keep last token to restore state
965              ParserSingleInstance::pushControlStatus(Parser::WithinDots);
966              yy_push_state(LINEBREAK);
967          }
968
969   <<EOF>>       {
970       yy_pop_state();
971       paren_levels.pop();
972   }
973 }
974
975 <LINEBREAK>
976 {
977   {newline}                             {
978     yylloc.last_line += 1;
979     yylloc.last_column = 1;
980     scan_step();
981     last_token = linebreak_stored_token;
982     if (linebreak_stored_space)
983     {
984         // This is important to restore coding spaces as if ... was not present
985         unput(' ');
986         linebreak_stored_space = FALSE;
987     }
988     yy_pop_state();
989     ParserSingleInstance::popControlStatus();
990   }
991
992   {startblockcomment}                   {
993     yylval.comment = new std::string();
994     comment_level = 1;
995     ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
996     yy_push_state(REGIONCOMMENT);
997   }
998
999   {startcomment}                        {
1000     scan_throw(DOTS);
1001     pstBuffer.clear();
1002     yy_push_state(LINECOMMENT);
1003   }
1004
1005   {spaces}                              {
1006       /* Do nothing... */
1007   }
1008
1009   <<EOF>>       {
1010       yy_pop_state();
1011   }
1012   .                                     {
1013     // The following case is not handled by the parser
1014     // a line of code ... /* some multiline
1015     // comments */ continued here;
1016     // without the special case telling we are after comments
1017     // will generate the error as follows:
1018
1019     // Any characters after ... yields to an error
1020     std::string str = "Unexpected token \'";
1021     str += yytext;
1022     str += "\' after line break with .. or ...";
1023     yy_pop_state();
1024     ParserSingleInstance::popControlStatus();
1025     BEGIN(INITIAL);
1026     yyerror(str);
1027     return scan_throw(FLEX_ERROR);
1028   }
1029 }
1030
1031
1032 <LINECOMMENT>
1033 {
1034   {newline}     {
1035     //yylloc.last_line += 1;
1036     //yylloc.last_column = 1;
1037     //scan_step();
1038     yy_pop_state();
1039     for (int i = yyleng - 1 ; i >= 0 ; --i)
1040     {
1041         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
1042         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
1043         unput(yytext[i]);
1044         yylloc.last_column--;
1045     }
1046     /*
1047     ** To forgot comments after lines break
1048     */
1049     if (last_token != DOTS)
1050     {
1051         yylval.comment = new std::string(pstBuffer);
1052         pstBuffer.clear();
1053         return scan_throw(COMMENT);
1054     }
1055     else
1056     {
1057         pstBuffer.clear();
1058     }
1059   }
1060
1061   <<EOF>>       {
1062     yy_pop_state();
1063     yylval.comment = new std::string(pstBuffer);
1064     pstBuffer.clear();
1065     return scan_throw(COMMENT);
1066   }
1067
1068   {char_in_line_comment}         {
1069       // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
1070       // only translate to WCHAR_T when popping state.
1071       pstBuffer += yytext;
1072   }
1073 }
1074
1075
1076 <REGIONCOMMENT>
1077 {
1078   {endblockcomment}                             {
1079     --comment_level;
1080     if (comment_level == 0) {
1081       ParserSingleInstance::popControlStatus();
1082       yy_pop_state();
1083       //return scan_throw(BLOCKCOMMENT);
1084     }
1085   }
1086
1087   {startblockcomment}                           {
1088     ++comment_level;
1089     yy_push_state(REGIONCOMMENT);
1090   }
1091
1092   {newline}                                     {
1093     yylloc.last_line += 1;
1094     yylloc.last_column = 1;
1095     scan_step();
1096     *yylval.comment += "\n//";
1097   }
1098
1099   {char_in_comment}                             |
1100   .                                             {
1101       *yylval.comment += std::string(yytext);
1102   }
1103
1104  <<EOF>>                                        {
1105       yy_pop_state();
1106 //    std::string str = "unexpected end of file in a comment";
1107 //    scan_error(str);
1108   }
1109 }
1110
1111
1112 <SIMPLESTRING>
1113 {
1114   {dquote}{dquote}                              {
1115     pstBuffer += "\"";
1116   }
1117
1118   {dquote}{quote}                               {
1119     pstBuffer += "'";
1120   }
1121
1122   {quote}{dquote}                               {
1123     pstBuffer += "\"";
1124   }
1125
1126   {quote}{quote}                                {
1127     pstBuffer += "'";
1128   }
1129
1130   {quote}                                       {
1131     yy_pop_state();
1132     //scan_step();
1133     yylval.str = new std::string(pstBuffer);
1134     pstBuffer.clear();
1135     yylloc.first_column = str_opener_column;
1136     return scan_throw(STR);
1137   }
1138
1139   {dquote}                  {
1140     pstBuffer.clear();
1141     BEGIN(INITIAL);
1142     yyerror("Heterogeneous string detected, starting with \' and ending with \".");
1143     return scan_throw(FLEX_ERROR);
1144   }
1145
1146   {newline}                                     {
1147     pstBuffer.clear();
1148     yylloc.last_line += 1;
1149     yylloc.last_column = 1;
1150     BEGIN(INITIAL);
1151     ParserSingleInstance::popControlStatus();
1152     yyerror("Unexpected end of line in a string.");
1153     return scan_throw(FLEX_ERROR);
1154   }
1155
1156   <<EOF>>                                       {
1157     pstBuffer.clear();
1158     BEGIN(INITIAL);
1159     yyerror("Unexpected end of file in a string.");
1160     return scan_throw(FLEX_ERROR);
1161   }
1162
1163   {in_string}                                           |
1164   .                                                     {
1165     //scan_step();
1166     pstBuffer += yytext;
1167   }
1168 }
1169
1170
1171 <DOUBLESTRING>
1172 {
1173   {dquote}{dquote}                              {
1174     pstBuffer += "\"";
1175   }
1176
1177   {dquote}{quote}                               {
1178     pstBuffer += "'";
1179   }
1180
1181   {quote}{dquote}               {
1182     pstBuffer += "\"";
1183   }
1184
1185   {quote}{quote}                                {
1186     pstBuffer += "'";
1187   }
1188
1189   {dquote}                      {
1190     yy_pop_state();
1191     //scan_step();
1192     yylval.str = new std::string(pstBuffer);
1193     pstBuffer.clear();
1194     yylloc.first_column = str_opener_column;
1195     return scan_throw(STR);
1196   }
1197
1198   {quote}                  {
1199     pstBuffer.clear();
1200     BEGIN(INITIAL);
1201     yyerror("Heterogeneous string detected, starting with \" and ending with \'.");
1202     return scan_throw(FLEX_ERROR);
1203   }
1204
1205   {newline} {
1206     pstBuffer.clear();
1207     yylloc.last_line += 1;
1208     yylloc.last_column = 1;
1209     ParserSingleInstance::popControlStatus();
1210     BEGIN(INITIAL);
1211     yyerror("Unexpected end of line in a string.");
1212     return scan_throw(FLEX_ERROR);
1213   }
1214
1215   <<EOF>>   {
1216     pstBuffer.clear();
1217     BEGIN(INITIAL);
1218     yyerror("Unexpected end of file in a string.");
1219     return scan_throw(FLEX_ERROR);
1220   }
1221
1222   {in_string}         |
1223   .                   {
1224    //scan_step();
1225    pstBuffer += yytext;
1226   }
1227 }
1228
1229
1230 <SHELLMODE>
1231 {
1232     {spaces}                    {
1233         if (last_token == ID)
1234         {
1235             scan_throw(SPACES);
1236             //return ID;
1237         }
1238     }
1239
1240     {semicolon}                 {
1241         BEGIN(INITIAL);
1242         scan_step();
1243         return scan_throw(SEMI);
1244     }
1245
1246     {comma}                     {
1247         BEGIN(INITIAL);
1248         scan_step();
1249         return scan_throw(COMMA);
1250     }
1251
1252     {newline}                   {
1253         BEGIN(INITIAL);
1254         yylloc.last_line += 1;
1255         yylloc.last_column = 1;
1256         scan_step();
1257         return scan_throw(EOL);
1258     }
1259
1260     {assign} {
1261         if (last_token == STR || last_token == SPACES)
1262         {
1263             yylval.str = new std::string(yytext);
1264             return scan_throw(STR);
1265         }
1266         else
1267         {
1268             BEGIN(INITIAL);
1269             return scan_throw(ASSIGN);
1270         }
1271     }
1272
1273     {lparen} {
1274         if (last_token == STR || last_token == SPACES)
1275         {
1276             yylval.str = new std::string(yytext);
1277             return scan_throw(STR);
1278         }
1279         else
1280         {
1281             BEGIN(INITIAL);
1282             return scan_throw(LPAREN);
1283         }
1284     }
1285
1286     {lowerthan} {
1287         if (last_token == STR || last_token == SPACES)
1288         {
1289             yylval.str = new std::string(yytext);
1290             return scan_throw(STR);
1291         }
1292         else
1293         {
1294             BEGIN(INITIAL);
1295             return scan_throw(LT);
1296         }
1297     }
1298
1299     {greaterthan} {
1300         if (last_token == STR || last_token == SPACES)
1301         {
1302             yylval.str = new std::string(yytext);
1303             return scan_throw(STR);
1304         }
1305         else
1306         {
1307             BEGIN(INITIAL);
1308             return scan_throw(GT);
1309         }
1310     }
1311
1312     {boolnot} {
1313         if (last_token == STR || last_token == SPACES)
1314         {
1315             yylval.str = new std::string(yytext);
1316             return scan_throw(STR);
1317         }
1318         else
1319         {
1320             BEGIN(INITIAL);
1321             return scan_throw(NOT);
1322         }
1323     }
1324
1325
1326     [^ \t\v\f\r\n,;'"]+               {
1327         yylval.str = new std::string(yytext);
1328         return scan_throw(STR);
1329     }
1330
1331     <<EOF>>                     {
1332         BEGIN(INITIAL);
1333     }
1334
1335 }
1336
1337 %%
1338
1339 int scan_throw(int token) {
1340   last_token = token;
1341 #ifdef DEV
1342   std::cout << "--> [DEBUG] TOKEN : " << token << " - " << token_to_string(token) << std::endl;
1343 #endif
1344   return token;
1345 }
1346
1347 int get_last_token() {
1348     return last_token;
1349 }
1350
1351 void scan_step() {
1352   yylloc.first_line = yylloc.last_line;
1353   yylloc.first_column = yylloc.last_column;
1354 }
1355
1356 /*
1357 ** convert floating numbers to C standard
1358 ** 1.2d-3 -> 1.2e-3
1359 ** 1.2D-3 -> 1.2e-3
1360 */
1361 void scan_exponent_convert(char *in)
1362 {
1363   for (; *in != 'd' && *in != 'D'; ++in);
1364   *in = 'e';
1365 }
1366
1367 #ifdef _MSC_VER
1368 int isatty (int desc)
1369 {
1370   return 0;
1371 }
1372 #endif
1373
1374 #ifdef DEV
1375 std::string token_to_string(int token)
1376 {
1377     std::string str;
1378     switch(token)
1379     {
1380         case AND :                   str = "AND";
1381                                      break;
1382         case ASSIGN :                str = "ASSIGN";
1383                                      break;
1384         case BOOLFALSE :             str = "BOOLFALSE";
1385                                      break;
1386         case BOOLTRUE :              str = "BOOLTRUE";
1387                                      break;
1388         case BREAK :                 str = "BREAK";
1389                                      break;
1390         case CASE :                  str = "CASE";
1391                                      break;
1392         case CATCH :                 str = "CATCH";
1393                                      break;
1394         case COLON :                 str = "COLON";
1395                                      break;
1396         case COMMA :                 str = "COMMA";
1397                                      break;
1398         case COMMENT :               str = "COMMENT";
1399                                      break;
1400         case CONTINUE :              str = "CONTINUE";
1401                                      break;
1402         case CONTROLLDIVIDE :        str = "CONTROLLDIVIDE";
1403                                      break;
1404         case CONTROLRDIVIDE :        str = "CONTROLRDIVIDE";
1405                                      break;
1406         case CONTROLTIMES :          str = "CONTROLTIMES";
1407                                      break;
1408         case DO :                    str = "DO";
1409                                      break;
1410         case DOLLAR :                str = "DOLLAR";
1411                                      break;
1412         case DOT :                   str = "DOT";
1413                                      break;
1414         case DOTLDIVIDE :            str = "DOTLDIVIDE";
1415                                      break;
1416         case DOTPOWER :              str = "DOTPOWER";
1417                                      break;
1418         case DOTQUOTE :              str = "DOTQUOTE";
1419                                      break;
1420         case DOTRDIVIDE :            str = "DOTRDIVIDE";
1421                                      break;
1422         case DOTS :                  str = "DOTS";
1423                                      break;
1424         case DOTTIMES :              str = "DOTTIMES";
1425                                      break;
1426         case ELSE :                  str = "ELSE";
1427                                      break;
1428         case ELSEIF :                str = "ELSEIF";
1429                                      break;
1430         case END :                   str = "END";
1431                                      break;
1432         case ENDFUNCTION :           str = "ENDFUNCTION";
1433                                      break;
1434         case EOL :                   str = "EOL";
1435                                      break;
1436         case EQ :                    str = "EQ";
1437                                      break;
1438         case FLEX_ERROR :            str = "FLEX_ERROR";
1439                                      break;
1440         case FOR :                   str = "FOR";
1441                                      break;
1442         case FUNCTION :              str = "FUNCTION";
1443                                      break;
1444         case GE :                    str = "GE";
1445                                      break;
1446         case GT :                    str = "GT";
1447                                      break;
1448         case ID :                    str = "ID";
1449                                      break;
1450         case IF :                    str = "IF";
1451                                      break;
1452         case KRONLDIVIDE :           str = "KRONLDIVIDE";
1453                                      break;
1454         case KRONRDIVIDE :           str = "KRONRDIVIDE";
1455                                      break;
1456         case KRONTIMES :             str = "KRONTIMES";
1457                                      break;
1458         case LBRACE :                str = "LBRACE";
1459                                      break;
1460         case LBRACK :                str = "LBRACK";
1461                                      break;
1462         case LDIVIDE :               str = "LDIVIDE";
1463                                      break;
1464         case LE :                    str = "LE";
1465                                      break;
1466         case LPAREN :                str = "LPAREN";
1467                                      break;
1468         case LT :                    str = "LT";
1469                                      break;
1470         case MINUS :                 str = "MINUS";
1471                                      break;
1472         case NE :                    str = "NE";
1473                                      break;
1474         case NOT :                   str = "NOT";
1475                                      break;
1476         case NUM :                   str = "NUM";
1477                                      break;
1478         case OR :                    str = "OR";
1479                                      break;
1480         case OROR :                  str = "OROR";
1481                                      break;
1482         case OTHERWISE :             str = "OTHERWISE";
1483                                      break;
1484         case PLUS :                  str = "PLUS";
1485                                      break;
1486         case POWER :                 str = "POWER";
1487                                      break;
1488         case QUOTE :                 str = "QUOTE";
1489                                      break;
1490         case RBRACE :                str = "RBRACE";
1491                                      break;
1492         case RBRACK :                str = "RBRACK";
1493                                      break;
1494         case RDIVIDE :               str = "RDIVIDE";
1495                                      break;
1496         case RETURN :                str = "RETURN";
1497                                      break;
1498         case RPAREN :                str = "RPAREN";
1499                                      break;
1500         case SELECT :                str = "SELECT";
1501                                      break;
1502         case SEMI :                  str = "SEMI";
1503                                      break;
1504         case SPACES :                str = "SPACES";
1505                                      break;
1506         case STR :                   str = "STR";
1507                                      break;
1508         case SWITCH :                str = "SWITCH";
1509                                      break;
1510         case THEN :                  str = "THEN";
1511                                      break;
1512         case TIMES :                 str = "TIMES";
1513                                      break;
1514         case TRY :                   str = "TRY";
1515                                      break;
1516         case VARFLOAT :              str = "VARFLOAT";
1517                                      break;
1518         case VARINT :                str = "VARINT";
1519                                      break;
1520         case WHILE :                 str = "WHILE";
1521                                      break;
1522         default :                    str = "UNKNOWN";
1523                                      break;
1524     }
1525     return str;
1526
1527 }
1528 #endif