License Header change: Removed the LICENSE_END before beta
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  * Copyright (C) 2012 - 2016 - Scilab Enterprises
7  *
8  * This file is hereby licensed under the terms of the GNU GPL v2.0,
9  * pursuant to article 5.3.4 of the CeCILL v.2.1.
10  * This file was originally licensed under the terms of the CeCILL v2.1,
11  * and continues to be available under such terms.
12  * For more information, see the COPYING file which you should have received
13  * along with this program.
14  *
15  */
16
17 #include <stack>
18
19 #include "isatty.hxx"
20 #include "parse.hxx"
21 #include "parser_private.hxx"
22
23 #include "context.hxx"
24
25 extern "C"
26 {
27 #include "charEncoding.h"
28 #include "sci_malloc.h"
29 }
30
31 static std::stack<int> paren_levels;
32
33 static int comment_level = 0;
34 static int last_token = 0;
35 static int exit_status = PARSE_ERROR;
36 static int str_opener_column = 0;
37 static std::string current_file;
38 static std::string program_name;
39
40 static std::string pstBuffer;
41
42 extern void yyerror(std::string);
43
44 #define YY_USER_ACTION                          \
45     yylloc.first_column = yylloc.last_column; yylloc.last_column += yyleng;
46 //yylloc.last_column += yyleng;
47
48 /* -*- Verbose Special Debug -*- */
49 //#define DEV
50 //#define TOKENDEV
51
52 #ifdef DEV
53 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
54 #else
55 #define DEBUG(x) /* Nothing */
56 #endif
57
58 %}
59
60 %option stack
61 %option noyywrap
62
63 %x SIMPLESTRING
64 %x DOUBLESTRING
65 %x REGIONCOMMENT
66 %x LINECOMMENT
67 %x LINEBREAK
68
69 %x MATRIX
70
71 %x SHELLMODE
72 %x BEGINID
73
74 spaces                  [ \t\v\f]+
75 integer                 [0-9]+
76 number                  [0-9]+[\.][0-9]*
77 little                  \.[0-9]+
78 bom             \xEF\xBB\xBF
79
80 floating_D              ({little}|{number}|{integer})[dD][+-]?{integer}
81 floating_E              ({little}|{number}|{integer})[eE][+-]?{integer}
82
83 hex             [0]x[0-9a-fA-F]+
84 oct             [0]o[0-7]+
85
86
87 utf2            ([\xC2-\xDF][\x80-\xBF])
88 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
89 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
90 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
91 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
92 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
93 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
94 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
95
96 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
97 utf4            ({utf41}|{utf42}|{utf43})
98
99 utf             ({utf2}|{utf3}|{utf4})
100 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
101
102
103 newline                 ("\r"|"\n"|"\r\n")
104 blankline               {spaces}+{newline}
105 emptyline       {newline}({spaces}|[,;])+{newline}
106 next                    \.\.+
107 char_in_line_comment    [^\r\n]*
108 char_in_comment         [^\r\n\/*]*
109
110 boolnot                 ("@"|"~")
111 booltrue                ("%t"|"%T")
112 boolfalse               ("%f"|"%F")
113 booland                 ("&")
114 boolandand              ("&&")
115 boolor                  ("|")
116 booloror                ("||")
117
118 lbrack                  "["
119 rbrack                  "]"
120
121 lparen                  "("
122 rparen                  ")"
123
124 lbrace                  "{"
125 rbrace                  "}"
126
127 dollar                  "$"
128
129 semicolon               ";"
130 comma                   ","
131 colon                   ":"
132
133 startcomment            "//"
134 startblockcomment       "/*"
135 endblockcomment         "*/"
136
137 dquote                  "\""
138 quote                   "'"
139 in_string               [^\"\'\r\n\.]*
140
141 dot             "."
142 dotquote                ".'"
143 dottimes                ".*"
144 dotrdivide              "./"
145 dotldivide              ".\\"
146 dotpower                (".^"|".**")
147
148 plus                    "+"
149 minus                   "-"
150 rdivide                 "/"
151 ldivide                 "\\"
152 times                   "*"
153 power                   ("^"|"**")
154
155 equal                   "="{spaces}*"="
156 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
157 lowerthan               "<"
158 greaterthan             ">"
159 lowerequal              "<"{spaces}*"="
160 greaterequal            ">"{spaces}*"="
161
162 krontimes               ".*."
163 kronrdivide             "./."
164 kronldivide             ".\\."
165
166 controltimes    ("*."[^0-9])
167 controlrdivide  ("/."[^0-9])
168 controlldivide  ("\\."[^0-9])
169
170 assign                  "="
171
172 %%
173 <INITIAL>{bom}/.* {
174 // BOM found ==> ignored
175 }
176
177 <INITIAL,BEGINID>{booltrue}     {
178     BEGIN(INITIAL);
179     return scan_throw(BOOLTRUE);
180 }
181 <INITIAL,BEGINID>{boolfalse}    {
182     BEGIN(INITIAL);
183     return scan_throw(BOOLFALSE);
184 }
185
186 <INITIAL,BEGINID>"if"            {
187         if (last_token != DOT)
188     {
189         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
190     }
191     DEBUG("BEGIN(INITIAL)");
192     BEGIN(INITIAL);
193     return scan_throw(IF);
194 }
195
196 <INITIAL,BEGINID>"then"          {
197     DEBUG("BEGIN(INITIAL)");
198     BEGIN(INITIAL);
199     return scan_throw(THEN);
200 }
201
202 <INITIAL,BEGINID>"else"          {
203         if (last_token != DOT)
204     {
205         // Pop to step out IF
206         ParserSingleInstance::popControlStatus();
207         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
208     }
209     DEBUG("BEGIN(INITIAL)");
210     BEGIN(INITIAL);
211         return scan_throw(ELSE);
212 }
213
214 <INITIAL,BEGINID>"elseif" {
215         if (last_token != DOT)
216     {
217         ParserSingleInstance::popControlStatus();
218         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
219     }
220     DEBUG("BEGIN(INITIAL)");
221     BEGIN(INITIAL);
222         return scan_throw(ELSEIF);
223 }
224
225 <INITIAL,BEGINID>"end"          {
226         if (last_token != DOT)
227     {
228         ParserSingleInstance::popControlStatus();
229     }
230     DEBUG("BEGIN(INITIAL)");
231     BEGIN(INITIAL);
232     return scan_throw(END);
233 }
234
235 <INITIAL,BEGINID>"select"       {
236         if (last_token != DOT)
237     {
238         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
239     }
240     DEBUG("BEGIN(INITIAL)");
241     BEGIN(INITIAL);
242     return scan_throw(SELECT);
243 }
244
245 <INITIAL,BEGINID>"switch"       {
246         if (last_token != DOT)
247     {
248         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
249     }
250     DEBUG("BEGIN(INITIAL)");
251     BEGIN(INITIAL);
252     return scan_throw(SWITCH);
253 }
254
255 <INITIAL,BEGINID>"otherwise" {
256         if (last_token != DOT)
257     {
258         ParserSingleInstance::popControlStatus();
259         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
260     }
261     DEBUG("BEGIN(INITIAL)");
262     BEGIN(INITIAL);
263         return scan_throw(OTHERWISE);
264 }
265
266 <INITIAL,BEGINID>"case"         {
267         if (last_token != DOT)
268     {
269         ParserSingleInstance::popControlStatus();
270         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
271     }
272     DEBUG("BEGIN(INITIAL)");
273     BEGIN(INITIAL);
274     return scan_throw(CASE);
275 }
276
277 <INITIAL,BEGINID>"function" {
278         if (last_token != DOT)
279     {
280         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
281     }
282     DEBUG("BEGIN(INITIAL)");
283     BEGIN(INITIAL);
284     return scan_throw(FUNCTION);
285 }
286
287 <INITIAL,BEGINID>"endfunction" {
288         if (last_token != DOT)
289     {
290         ParserSingleInstance::popControlStatus();
291     }
292     DEBUG("BEGIN(INITIAL)");
293     BEGIN(INITIAL);
294         return scan_throw(ENDFUNCTION);
295 }
296
297 <INITIAL,BEGINID>"for" {
298         if (last_token != DOT)
299     {
300         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
301     }
302     BEGIN(INITIAL);
303     return scan_throw(FOR);
304 }
305
306 <INITIAL,BEGINID>"while"        {
307         if (last_token != DOT)
308     {
309         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
310     }
311         BEGIN(INITIAL);
312         return scan_throw(WHILE);
313 }
314
315 <INITIAL,BEGINID>"do"           {
316         BEGIN(INITIAL);
317     return scan_throw(DO);
318 }
319
320 <INITIAL,BEGINID>"break"                {
321         BEGIN(INITIAL);
322         return scan_throw(BREAK);
323 }
324
325 <INITIAL,BEGINID>"continue"             {
326         BEGIN(INITIAL);
327         return scan_throw(CONTINUE);
328 }
329
330 <INITIAL,BEGINID>"try" {
331         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
332         BEGIN(INITIAL);
333         return scan_throw(TRY);
334 }
335
336 <INITIAL,BEGINID>"catch" {
337     // Pop to step out TRY
338         ParserSingleInstance::popControlStatus();
339         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
340         BEGIN(INITIAL);
341         return scan_throw(CATCH);
342 }
343
344 <INITIAL,BEGINID>"return"       {
345     BEGIN(INITIAL);
346     return scan_throw(RETURN);
347 }
348
349 <INITIAL,BEGINID>"resume"       {
350     BEGIN(INITIAL);
351     return scan_throw(RETURN);
352 }
353
354 ^{spaces}*/({id}){spaces}([^ \t\v\f(=<>~@]|([~@]{spaces}*[^=]?)) {
355         BEGIN(BEGINID);
356 }
357
358 <BEGINID>
359 {
360     {id}                        {
361         wchar_t *pwText = to_wide_string(yytext);
362         if (yytext != NULL && pwText == NULL)
363         {
364             std::string str = "Can\'t convert \'";
365             str += yytext;
366             str += "\' to UTF-8";
367             BEGIN(INITIAL);
368             yyerror(str);
369             return scan_throw(FLEX_ERROR);
370         }
371         yylval.str = new std::wstring(pwText);
372         FREE(pwText);
373         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
374         if (pIT && pIT->isCallable())
375         {
376             scan_throw(ID);
377             BEGIN(SHELLMODE);
378         }
379         else
380         {
381             BEGIN(INITIAL);
382             return scan_throw(ID);
383         }
384     }
385
386 }
387
388 <INITIAL,MATRIX>{boolnot}               {
389   return scan_throw(NOT);
390 }
391 <INITIAL,MATRIX>{dollar}                {
392   return scan_throw(DOLLAR);
393 }
394 <INITIAL,MATRIX>{booltrue}              {
395   return scan_throw(BOOLTRUE);
396 }
397 <INITIAL,MATRIX>{boolfalse}             {
398   return scan_throw(BOOLFALSE);
399 }
400 <INITIAL,MATRIX>{booland}               {
401   return scan_throw(AND);
402 }
403 <INITIAL,MATRIX>{boolandand}    {
404   return scan_throw(ANDAND);
405 }
406 <INITIAL,MATRIX>{boolor}                {
407   return scan_throw(OR);
408 }
409 <INITIAL,MATRIX>{booloror}              {
410   return scan_throw(OROR);
411 }
412
413
414 <INITIAL>{lparen}                       {
415   return scan_throw(LPAREN);
416 }
417 <INITIAL>{rparen}                       {
418   return scan_throw(RPAREN);
419 }
420
421
422 <INITIAL,MATRIX>{semicolon}             {
423         scan_step();
424   return scan_throw(SEMI);
425 }
426
427 <INITIAL,MATRIX>{comma}                 {
428         scan_step();
429   return scan_throw(COMMA);
430 }
431
432 <INITIAL,MATRIX>{colon}                 {
433   return scan_throw(COLON);
434 }
435
436
437 <INITIAL,MATRIX>{lbrace}                {
438   yy_push_state(MATRIX);
439   paren_levels.push(0);
440   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
441   return scan_throw(LBRACE);
442 }
443
444 {rbrace}                        {
445   return scan_throw(RBRACE);
446 }
447
448
449 <INITIAL,MATRIX>{dotquote}              {
450   return scan_throw(DOTQUOTE);
451 }
452 <INITIAL,MATRIX>{dottimes}              {
453   return scan_throw(DOTTIMES);
454 }
455 <INITIAL,MATRIX>{dotrdivide}            {
456   return scan_throw(DOTRDIVIDE);
457 }
458 <INITIAL,MATRIX>{dotldivide}            {
459   return scan_throw(DOTLDIVIDE);
460 }
461 <INITIAL,MATRIX>{dotpower}              {
462   return scan_throw(DOTPOWER);
463 }
464
465
466 {minus}                                 {
467   return scan_throw(MINUS);
468 }
469 {plus}                                  {
470   return scan_throw(PLUS);
471 }
472 <INITIAL,MATRIX>{times}                 {
473   return scan_throw(TIMES);
474 }
475 <INITIAL,MATRIX>{rdivide}               {
476   return scan_throw(RDIVIDE);
477 }
478 <INITIAL,MATRIX>{ldivide}               {
479   return scan_throw(LDIVIDE);
480 }
481 <INITIAL,MATRIX>{power}                 {
482   return scan_throw(POWER);
483 }
484
485 <INITIAL,MATRIX>{krontimes}             {
486   return scan_throw(KRONTIMES);
487 }
488 <INITIAL,MATRIX>{kronrdivide}           {
489   return scan_throw(KRONRDIVIDE);
490 }
491 <INITIAL,MATRIX>{kronldivide}           {
492   return scan_throw(KRONLDIVIDE);
493 }
494
495
496 <INITIAL,MATRIX>{controltimes}          {
497     unput(yytext[yyleng - 1]);
498     return scan_throw(CONTROLTIMES);
499 }
500 <INITIAL,MATRIX>{controlrdivide}                {
501     unput(yytext[yyleng - 1]);
502     return scan_throw(CONTROLRDIVIDE);
503 }
504 <INITIAL,MATRIX>{controlldivide}                {
505     unput(yytext[yyleng - 1]);
506     return scan_throw(CONTROLLDIVIDE);
507 }
508
509
510 <INITIAL,MATRIX>{equal}                 {
511   return scan_throw(EQ);
512 }
513 <INITIAL,MATRIX>{notequal}              {
514   return scan_throw(NE);
515 }
516 <INITIAL,MATRIX>{lowerthan}             {
517   return scan_throw(LT);
518 }
519 <INITIAL,MATRIX>{greaterthan}           {
520   return scan_throw(GT);
521 }
522 <INITIAL,MATRIX>{lowerequal}            {
523   return scan_throw(LE);
524 }
525 <INITIAL,MATRIX>{greaterequal}          {
526   return scan_throw(GE);
527 }
528
529
530 <INITIAL,MATRIX>{assign}                {
531   return scan_throw(ASSIGN);
532  }
533
534
535 <INITIAL,MATRIX>{lbrack}                {
536   DEBUG("yy_push_state(MATRIX)");
537   yy_push_state(MATRIX);
538   paren_levels.push(0);
539   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
540   return scan_throw(LBRACK);
541 }
542
543 <INITIAL>{rbrack}                               {
544   return scan_throw(RBRACK);
545 }
546
547
548 <INITIAL,MATRIX>{dot}                   {
549   return scan_throw(DOT);
550 }
551
552 <INITIAL>{next}                 {
553     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
554     yy_push_state(LINEBREAK);
555 }
556
557 <INITIAL,MATRIX>{integer}               {
558   yylval.number = atof(yytext);
559 #ifdef TOKENDEV
560   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
561 #endif
562 //  scan_step();
563   return scan_throw(VARINT);
564 }
565
566
567 <INITIAL,MATRIX>{floating_D}            {
568   scan_exponent_convert(yytext);
569   yylval.number = atof(yytext);
570 #ifdef TOKENDEV
571   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
572 #endif
573   //scan_step();
574   return scan_throw(VARFLOAT);
575 }
576
577 <INITIAL,MATRIX>{floating_E}            {
578   yylval.number = atof(yytext);
579 #ifdef TOKENDEV
580   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
581 #endif
582   //scan_step();
583   return scan_throw(VARFLOAT);
584 }
585
586 <INITIAL,MATRIX>[0-9]+[\.]/[\*^\\\/]            {
587
588   yylval.number = atof(yytext);
589 #ifdef TOKENDEV
590   std::cout << "--> [DEBUG] NUMBER WITH DOT AS LAST CHARACTER : " << yytext << std::endl;
591 #endif
592 //  scan_step();
593   unput('.');
594   yylloc.last_column--;
595   return scan_throw(NUM);
596 }
597
598 <INITIAL,MATRIX>{number}                {
599   yylval.number = atof(yytext);
600 #ifdef TOKENDEV
601   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
602 #endif
603 //  scan_step();
604   return scan_throw(NUM);
605 }
606
607
608 <INITIAL,MATRIX>{little}                {
609   yylval.number = atof(yytext);
610 #ifdef TOKENDEV
611   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
612 #endif
613 //  scan_step();
614   return scan_throw(NUM);
615 }
616
617
618 <INITIAL,MATRIX>{id}                    {
619     wchar_t *pwText = to_wide_string(yytext);
620     if (yytext != NULL && pwText == NULL)
621     {
622         std::string str = "Can\'t convert \'";
623         str += yytext;
624         str += "\' to UTF-8";
625         BEGIN(INITIAL);
626         yyerror(str);
627         return scan_throw(FLEX_ERROR);
628     }
629     yylval.str = new std::wstring(pwText);
630     FREE(pwText);
631 #ifdef TOKENDEV
632   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
633 #endif
634 //  scan_step();
635   return scan_throw(ID);
636 }
637
638
639 <INITIAL,MATRIX>{startblockcomment}     {
640   yylval.comment = new std::wstring();
641   comment_level = 1;
642   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
643   yy_push_state(REGIONCOMMENT);
644 }
645
646
647 <INITIAL,MATRIX>{startcomment}          {
648   pstBuffer.clear();
649   yy_push_state(LINECOMMENT);
650 }
651
652
653 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
654   pstBuffer.clear();
655   str_opener_column = yylloc.first_column;
656   yy_push_state(DOUBLESTRING);
657 }
658
659
660 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
661   /*
662   ** Matrix Transposition special behaviour
663   ** ID' []' toto()' are transposition call
664   */
665   if (last_token == ID
666       || last_token == RBRACK
667       || last_token == RPAREN
668       || last_token == RBRACE
669       || last_token == VARINT
670       || last_token == VARFLOAT
671       || last_token == NUM
672       || last_token == BOOLTRUE
673       || last_token == BOOLFALSE)
674   {
675       return scan_throw(QUOTE);
676   }
677   else
678   {
679       pstBuffer.clear();
680       str_opener_column = yylloc.first_column;
681       yy_push_state(SIMPLESTRING);
682   }
683 }
684
685
686 <INITIAL,MATRIX>{spaces}                {
687         scan_step();
688         scan_throw(SPACES);
689 }
690
691
692 <INITIAL>{newline}              {
693   yylloc.last_line += 1;
694   yylloc.last_column = 1;
695   scan_step();
696   if (last_token != EOL) {
697       return scan_throw(EOL);
698   }
699
700 }
701
702
703 <INITIAL,MATRIX>{blankline}             {
704   yylloc.last_line += 1;
705   yylloc.last_column = 1;
706   scan_step();
707   if (last_token != EOL)
708   {
709       return scan_throw(EOL);
710   }
711   scan_throw(EOL);
712 }
713
714 <INITIAL,MATRIX>{emptyline}             {
715   yylloc.last_line += 2;
716   yylloc.last_column = 1;
717   scan_step();
718   if (last_token != EOL)
719   {
720       return scan_throw(EOL);
721   }
722   scan_throw(EOL);
723 }
724 .                                       {
725     std::string str = "Unexpected token \'";
726     str += yytext;
727     str += "\'";
728     BEGIN(INITIAL);
729     yyerror(str);
730     return scan_throw(FLEX_ERROR);
731 }
732
733
734 <MATRIX>
735 {
736   {lparen} {
737     ++paren_levels.top();
738     return scan_throw(LPAREN);
739   }
740
741   {rparen} {
742     --paren_levels.top();
743     return scan_throw(RPAREN);
744   }
745
746   {spaces}*{lparen} {
747       unput(yytext[yyleng -1]);
748       if (last_token == ID
749           || last_token == RPAREN
750           || last_token == QUOTE
751           || last_token == VARINT
752           || last_token == VARFLOAT
753           || last_token == NUM)
754       {
755           return scan_throw(COMMA);
756       }
757   }
758
759   {spaces}*{colon}{spaces}* {
760       return scan_throw(COLON);
761   }
762
763   {newline} {
764       yylloc.last_line += 1;
765       yylloc.last_column = 1;
766       if(last_token != DOTS && last_token != EOL)
767       {
768           return scan_throw(EOL);
769       }
770       scan_throw(EOL);
771   }
772
773   {rbrack}                              {
774     DEBUG("yy_pop_state()");
775     yy_pop_state();
776     paren_levels.pop();
777     ParserSingleInstance::popControlStatus();
778     return scan_throw(RBRACK);
779   }
780
781   {rbrace}                              {
782     yy_pop_state();
783     paren_levels.pop();
784     ParserSingleInstance::popControlStatus();
785     return scan_throw(RBRACE);
786   }
787
788   {plus}                                |
789   {spaces}{plus}{spaces}                {
790     return scan_throw(PLUS);
791   }
792
793   {minus}                               |
794   {spaces}{minus}{spaces}               {
795     return scan_throw(MINUS);
796   }
797
798   {spaces}{plus}                        {
799     // no need to unput the '+'
800     if (last_token != LBRACK
801        && last_token != EOL
802        && last_token != SEMI
803        && last_token != COMMA
804        && last_token != DOTTIMES
805        && last_token != DOTRDIVIDE
806        && last_token != DOTLDIVIDE
807        && last_token != DOTPOWER
808        && last_token != MINUS
809        && last_token != PLUS
810        && last_token != TIMES
811        && last_token != RDIVIDE
812        && last_token != LDIVIDE
813        && last_token != POWER
814        && last_token != KRONTIMES
815        && last_token != KRONRDIVIDE
816        && last_token != KRONLDIVIDE
817        && last_token != EQ
818        && last_token != NE
819        && last_token != LT
820        && last_token != GT
821        && last_token != LE
822        && last_token != GE
823       && paren_levels.top() == 0)
824    {
825        return scan_throw(COMMA);
826    }
827    else
828    {
829        unput('+');
830        yylloc.last_column--;
831    }
832   }
833
834   {spaces}{minus}                       {
835     unput('-');
836     yylloc.last_column--;
837     if (last_token != LBRACK
838        && last_token != EOL
839        && last_token != SEMI
840        && last_token != COMMA
841        && last_token != DOTTIMES
842        && last_token != DOTRDIVIDE
843        && last_token != DOTLDIVIDE
844        && last_token != DOTPOWER
845        && last_token != MINUS
846        && last_token != PLUS
847        && last_token != TIMES
848        && last_token != RDIVIDE
849        && last_token != LDIVIDE
850        && last_token != POWER
851        && last_token != KRONTIMES
852        && last_token != KRONRDIVIDE
853        && last_token != KRONLDIVIDE
854        && last_token != EQ
855        && last_token != NE
856        && last_token != LT
857        && last_token != GT
858        && last_token != LE
859        && last_token != GE
860        && paren_levels.top() == 0)
861    {
862        return scan_throw(COMMA);
863    }
864   }
865
866   .                                     {
867     std::string str = "Unexpected token \'";
868     str += yytext;
869     str += "\' within a matrix.";
870     BEGIN(INITIAL);
871     yyerror(str);
872     return scan_throw(FLEX_ERROR);
873   }
874
875   {next}{spaces}*{newline}          {
876       yylloc.last_line += 1;
877       yylloc.last_column = 1;
878       scan_step();
879   }
880
881   {next}{spaces}*{startcomment}.*{newline}          {
882       yylloc.last_line += 1;
883       yylloc.last_column = 1;
884       scan_step();
885   }
886
887   {spaces}{next}{spaces}*{newline}          {
888       yylloc.last_line += 1;
889       yylloc.last_column = 1;
890       scan_step();
891       unput(' ');
892       yylloc.last_column--;
893   }
894
895   {spaces}{next}{spaces}*{startcomment}.*{newline}          {
896       yylloc.last_line += 1;
897       yylloc.last_column = 1;
898       scan_step();
899       unput(' ');
900       yylloc.last_column--;
901   }
902
903   <<EOF>>       {
904       yy_pop_state();
905       paren_levels.pop();
906   }
907 }
908
909 <LINEBREAK>
910 {
911   {newline}                             {
912     yylloc.last_line += 1;
913     yylloc.last_column = 1;
914     scan_step();
915     yy_pop_state();
916     ParserSingleInstance::popControlStatus();
917   }
918
919   {startblockcomment}                   {
920     ++comment_level;
921     yy_push_state(REGIONCOMMENT);
922   }
923
924   {startcomment}                        {
925     scan_throw(DOTS);
926     pstBuffer.clear();
927     yy_push_state(LINECOMMENT);
928   }
929
930   {spaces}                              {
931       /* Do nothing... */
932   }
933
934   <<EOF>>       {
935       yy_pop_state();
936   }
937   .                                     {
938     std::string str = "Unexpected token \'";
939     str += yytext;
940     str += "\' after line break with .. or ...";
941     BEGIN(INITIAL);
942     yyerror(str);
943     return scan_throw(FLEX_ERROR);
944   }
945 }
946
947
948 <LINECOMMENT>
949 {
950   {newline}     {
951     //yylloc.last_line += 1;
952     //yylloc.last_column = 1;
953     //scan_step();
954     yy_pop_state();
955     for (int i = yyleng - 1 ; i >= 0 ; --i)
956     {
957         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
958         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
959         unput(yytext[i]);
960         yylloc.last_column--;
961     }
962     /*
963     ** To forgot comments after lines break
964     */
965     if (last_token != DOTS)
966     {
967         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
968         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
969         wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
970         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
971         if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
972         {
973             pstBuffer.clear();
974             std::string str = "Can\'t convert \'";
975             str += pstBuffer.c_str();
976             str += "\' to UTF-8";
977             BEGIN(INITIAL);
978             yyerror(str);
979             return scan_throw(FLEX_ERROR);
980         }
981         yylval.comment = new std::wstring(pwstBuffer);
982         pstBuffer.clear();
983         FREE (pwstBuffer);
984         return scan_throw(COMMENT);
985     }
986     else
987     {
988         pstBuffer.clear();
989     }
990   }
991
992   <<EOF>>       {
993     yy_pop_state();
994     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
995     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
996     {
997         pstBuffer.clear();
998         std::string str = "Can\'t convert \'";
999         str += pstBuffer.c_str();
1000         str += "\' to UTF-8";
1001         BEGIN(INITIAL);
1002         yyerror(str);
1003         return scan_throw(FLEX_ERROR);
1004     }
1005     yylval.comment = new std::wstring(pwstBuffer);
1006     pstBuffer.clear();
1007     FREE (pwstBuffer);
1008     return scan_throw(COMMENT);
1009   }
1010
1011   {char_in_line_comment}         {
1012       // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
1013       // only translate to WCHAR_T when popping state.
1014       pstBuffer += yytext;
1015   }
1016 }
1017
1018
1019 <REGIONCOMMENT>
1020 {
1021   {endblockcomment}                             {
1022     --comment_level;
1023     if (comment_level == 0) {
1024       ParserSingleInstance::popControlStatus();
1025       yy_pop_state();
1026       //return scan_throw(BLOCKCOMMENT);
1027     }
1028   }
1029
1030   {startblockcomment}                           {
1031     ++comment_level;
1032     yy_push_state(REGIONCOMMENT);
1033   }
1034
1035   {newline}                                     {
1036     yylloc.last_line += 1;
1037     yylloc.last_column = 1;
1038     scan_step();
1039     *yylval.comment += L"\n//";
1040   }
1041
1042   {char_in_comment}                             |
1043   .                                             {
1044       wchar_t *pwText = to_wide_string(yytext);
1045       *yylval.comment += std::wstring(pwText);
1046       FREE(pwText);
1047   }
1048
1049  <<EOF>>                                        {
1050       yy_pop_state();
1051 //    std::string str = "unexpected end of file in a comment";
1052 //    scan_error(str);
1053   }
1054 }
1055
1056
1057 <SIMPLESTRING>
1058 {
1059   {dquote}{dquote}                              {
1060     pstBuffer += "\"";
1061   }
1062
1063   {dquote}{quote}                               {
1064     pstBuffer += "'";
1065   }
1066
1067   {quote}{dquote}                               {
1068     pstBuffer += "\"";
1069   }
1070
1071   {quote}{quote}                                {
1072     pstBuffer += "'";
1073   }
1074
1075   {quote}                                       {
1076     yy_pop_state();
1077     //scan_step();
1078     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1079     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1080     {
1081         pstBuffer.clear();
1082         std::string str = "Can\'t convert \'";
1083         str += pstBuffer.c_str();
1084         str += "\' to UTF-8";
1085         BEGIN(INITIAL);
1086         yyerror(str);
1087         return scan_throw(FLEX_ERROR);
1088     }
1089     yylval.str = new std::wstring(pwstBuffer);
1090     pstBuffer.clear();
1091     FREE(pwstBuffer);
1092     yylloc.first_column = str_opener_column;
1093     return scan_throw(STR);
1094   }
1095
1096   {dquote}                  {
1097     pstBuffer.clear();
1098     BEGIN(INITIAL);
1099     yyerror("Heterogeneous string detected, starting with \' and ending with \".");
1100     return scan_throw(FLEX_ERROR);
1101   }
1102
1103   {next}{newline}           {
1104       yylloc.last_line += 1;
1105       yylloc.last_column = 1;
1106       /* Do nothing... Just skip */
1107   }
1108
1109   {newline}                                     {
1110     pstBuffer.clear();
1111     yylloc.last_line += 1;
1112     yylloc.last_column = 1;
1113     BEGIN(INITIAL);
1114     yyerror("Unexpected end of line in a string.");
1115     return scan_throw(FLEX_ERROR);
1116   }
1117
1118   <<EOF>>                                       {
1119     pstBuffer.clear();
1120     BEGIN(INITIAL);
1121     yyerror("Unexpected end of file in a string.");
1122     return scan_throw(FLEX_ERROR);
1123   }
1124
1125   {in_string}                                           |
1126   .                                                     {
1127     //scan_step();
1128     pstBuffer += yytext;
1129   }
1130 }
1131
1132
1133 <DOUBLESTRING>
1134 {
1135   {dquote}{dquote}                              {
1136     pstBuffer += "\"";
1137   }
1138
1139   {dquote}{quote}                               {
1140     pstBuffer += "'";
1141   }
1142
1143   {quote}{dquote}               {
1144     pstBuffer += "\"";
1145   }
1146
1147   {quote}{quote}                                {
1148     pstBuffer += "'";
1149   }
1150
1151   {dquote}                      {
1152     yy_pop_state();
1153     //scan_step();
1154     wchar_t *pwstBuffer = to_wide_string(pstBuffer.c_str());
1155     if (pstBuffer.c_str() != NULL && pwstBuffer == NULL)
1156     {
1157         pstBuffer.clear();
1158         std::string str = "Can\'t convert \'";
1159         str += pstBuffer.c_str();
1160         str += "\' to UTF-8";
1161         BEGIN(INITIAL);
1162         yyerror(str);
1163         return scan_throw(FLEX_ERROR);
1164     }
1165     yylval.str = new std::wstring(pwstBuffer);
1166     pstBuffer.clear();
1167     FREE(pwstBuffer);
1168     yylloc.first_column = str_opener_column;
1169     return scan_throw(STR);
1170   }
1171
1172   {quote}                  {
1173     pstBuffer.clear();
1174     BEGIN(INITIAL);
1175     yyerror("Heterogeneous string detected, starting with \" and ending with \'.");
1176     return scan_throw(FLEX_ERROR);
1177   }
1178
1179   {next}{newline}           {
1180       yylloc.last_line += 1;
1181       yylloc.last_column = 1;
1182       /* Do nothing... Just skip */
1183   }
1184
1185   {newline} {
1186     pstBuffer.clear();
1187     yylloc.last_line += 1;
1188     yylloc.last_column = 1;
1189     BEGIN(INITIAL);
1190     yyerror("Unexpected end of line in a string.");
1191     return scan_throw(FLEX_ERROR);
1192   }
1193
1194   <<EOF>>   {
1195     pstBuffer.clear();
1196     BEGIN(INITIAL);
1197     yyerror("Unexpected end of file in a string.");
1198     return scan_throw(FLEX_ERROR);
1199   }
1200
1201   {in_string}         |
1202   .                   {
1203    //scan_step();
1204    pstBuffer += yytext;
1205   }
1206 }
1207
1208
1209 <SHELLMODE>
1210 {
1211     {spaces}                    {
1212         if (last_token == ID)
1213         {
1214             scan_throw(SPACES);
1215             return ID;
1216         }
1217     }
1218
1219     {semicolon}                 {
1220         BEGIN(INITIAL);
1221         scan_step();
1222         return scan_throw(SEMI);
1223     }
1224
1225     {comma}                     {
1226         BEGIN(INITIAL);
1227         scan_step();
1228         return scan_throw(COMMA);
1229     }
1230
1231     {newline}                   {
1232         BEGIN(INITIAL);
1233         yylloc.last_line += 1;
1234         yylloc.last_column = 1;
1235         scan_step();
1236         return scan_throw(EOL);
1237     }
1238
1239     {assign} {
1240         if (last_token == STR || last_token == SPACES)
1241         {
1242             wchar_t *pwText = to_wide_string(yytext);
1243             yylval.str = new std::wstring(pwText);
1244             FREE(pwText);
1245             return scan_throw(STR);
1246         }
1247         else
1248         {
1249             BEGIN(INITIAL);
1250             return scan_throw(ASSIGN);
1251         }
1252     }
1253
1254     {lparen} {
1255         if (last_token == STR || last_token == SPACES)
1256         {
1257             wchar_t *pwText = to_wide_string(yytext);
1258             yylval.str = new std::wstring(pwText);
1259             FREE(pwText);
1260             return scan_throw(STR);
1261         }
1262         else
1263         {
1264             BEGIN(INITIAL);
1265             return scan_throw(LPAREN);
1266         }
1267     }
1268
1269     {lowerthan} {
1270         if (last_token == STR || last_token == SPACES)
1271         {
1272             wchar_t *pwText = to_wide_string(yytext);
1273             yylval.str = new std::wstring(pwText);
1274             FREE(pwText);
1275             return scan_throw(STR);
1276         }
1277         else
1278         {
1279             BEGIN(INITIAL);
1280             return scan_throw(LT);
1281         }
1282     }
1283
1284     {greaterthan} {
1285         if (last_token == STR || last_token == SPACES)
1286         {
1287             wchar_t *pwText = to_wide_string(yytext);
1288             yylval.str = new std::wstring(pwText);
1289             FREE(pwText);
1290             return scan_throw(STR);
1291         }
1292         else
1293         {
1294             BEGIN(INITIAL);
1295             return scan_throw(GT);
1296         }
1297     }
1298
1299     {boolnot} {
1300         if (last_token == STR || last_token == SPACES)
1301         {
1302             wchar_t *pwText = to_wide_string(yytext);
1303             yylval.str = new std::wstring(pwText);
1304             FREE(pwText);
1305             return scan_throw(STR);
1306         }
1307         else
1308         {
1309             BEGIN(INITIAL);
1310             return scan_throw(NOT);
1311         }
1312     }
1313
1314
1315     [^ \t\v\f\r\n,;'"]+               {
1316         wchar_t *pwText = to_wide_string(yytext);
1317         yylval.str = new std::wstring(pwText);
1318         FREE(pwText);
1319         return scan_throw(STR);
1320     }
1321
1322     <<EOF>>                     {
1323         BEGIN(INITIAL);
1324     }
1325
1326 }
1327
1328 %%
1329
1330 int scan_throw(int token) {
1331   last_token = token;
1332 #ifdef DEV
1333   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1334 #endif
1335   return token;
1336 }
1337
1338 int get_last_token() {
1339     return last_token;
1340 }
1341
1342 void scan_step() {
1343   yylloc.first_line = yylloc.last_line;
1344   yylloc.first_column = yylloc.last_column;
1345 }
1346
1347 /*
1348 ** convert floating numbers to C standard
1349 ** 1.2d-3 -> 1.2e-3
1350 ** 1.2D-3 -> 1.2e-3
1351 */
1352 void scan_exponent_convert(char *in)
1353 {
1354   for (; *in != 'd' && *in != 'D'; ++in);
1355   *in = 'e';
1356 }
1357
1358 #ifdef _MSC_VER
1359 int isatty (int desc)
1360 {
1361   return 0;
1362 }
1363 #endif