Bugs 10827,10868 fixed: In SciNotes, string containing LaTeX was removed on execution...
[scilab.git] / scilab / modules / scinotes / src / java / jflex / scilab.jflex
1 //CHECKSTYLE:OFF
2
3 package org.scilab.modules.scinotes;
4
5 import java.util.Arrays;
6 import java.util.HashSet;
7 import java.util.Iterator;
8 import java.util.ArrayList;
9 import java.util.List;
10 import java.util.Set;
11 import java.io.IOException;
12 import javax.swing.text.BadLocationException;
13 import javax.swing.text.Element;
14
15 import org.scilab.modules.commons.ScilabCommonsUtils;
16
17 @javax.annotation.Generated("JFlex")
18 @SuppressWarnings("fallthrough")
19
20 %%
21
22 %public
23 %class ScilabLexer
24 %extends ScilabLexerConstants
25 %final
26 %unicode
27 %char
28 %type int
29 %switch
30
31 %{
32     public int start;
33     public int end;
34     public int beginString;
35     public static Set<String> commands = new HashSet<String>();
36     public static Set<String> macros = new HashSet<String>();
37     public static Set<String> variables = new HashSet<String>();
38     public Set<String> infile;
39
40     private ScilabDocument doc;
41     private boolean transposable;
42     private Element elem;
43     private boolean breakstring;
44
45     public ScilabLexer(ScilabDocument doc) {
46         this.doc = doc;
47         this.elem = doc.getDefaultRootElement();
48         this.infile = doc.getFunctionsInDoc();
49         update();
50     }
51
52     public static void update() {
53         if (ScilabCommonsUtils.isScilabThread()) {
54             String[] vars = ScilabKeywords.GetVariablesName();
55             String[] funs = ScilabKeywords.GetFunctionsName();
56             String[] macs = ScilabKeywords.GetMacrosName();
57             variables.clear();
58             commands.clear();
59             macros.clear();
60             if (vars != null) {
61                 variables.addAll(Arrays.asList(vars));
62             }
63             if (funs != null) {
64                 commands.addAll(Arrays.asList(funs));
65             }
66             if (macs != null) {
67                 macros.addAll(Arrays.asList(macs));
68             }
69         }
70     }
71
72     public void setRange(int p0, int p1) {
73         start = p0;
74         end = p1;
75         transposable = false;
76         breakstring = false;
77         yyreset(new ScilabDocumentReader(doc, p0, p1));
78         int currentLine = elem.getElementIndex(start);
79         if (currentLine != 0 && ((ScilabDocument.ScilabLeafElement) elem.getElement(currentLine - 1)).isBrokenString()) {
80            yybegin(QSTRING);
81         }
82     }
83
84     public int yychar() {
85         return yychar;
86     }
87
88     public int scan() throws IOException {
89         int ret = yylex();
90         if (start + yychar + yylength() == end - 1) {
91            ((ScilabDocument.ScilabLeafElement) elem.getElement(elem.getElementIndex(start))).setBrokenString(breakstring);
92            breakstring = false;
93         }
94         return ret;
95     }
96
97     public int getKeyword(int pos, boolean strict) {
98         Element line = elem.getElement(elem.getElementIndex(pos));
99         int end = line.getEndOffset();
100         int tok = -1;
101         start = line.getStartOffset();
102         int startL = start;
103         int s = -1;
104
105         try {
106            yyreset(new ScilabDocumentReader(doc, start, end));
107            if (!strict) {
108               pos++;
109            }
110
111            while (startL < pos && (s != startL || yystate() == BREAKSTRING)) {
112                s = startL;
113                tok = yylex();
114                startL = start + yychar + yylength();
115            }
116
117            return tok;
118         } catch (IOException e) {
119            return ScilabLexerConstants.DEFAULT;
120         }
121      }
122
123      public static ScilabTokens getScilabTokens(String str) {
124          ScilabDocument doc = new ScilabDocument(false);
125          try {
126              doc.insertString(0, str, null);
127          } catch (BadLocationException e) { }
128          return getScilabTokens(doc);
129      }
130
131      public static ScilabTokens getScilabTokens(ScilabDocument doc) {
132          ScilabLexer lexer = new ScilabLexer(doc);
133          lexer.yyreset(new ScilabDocumentReader(doc, 0, doc.getLength()));
134          ScilabTokens tokens = new ScilabTokens();
135          int tok = -1;
136          try {
137             while (tok != ScilabLexerConstants.EOF) {
138                tok = lexer.yylex();
139                tokens.add(tok, lexer.yychar + lexer.yylength());
140             }
141          } catch (IOException e) { }
142  
143          return tokens;
144      }
145
146      public static class ScilabTokens {
147         private List<Integer> tokenType = new ArrayList<Integer>();
148         private List<Integer> tokenPos = new ArrayList<Integer>();
149
150         ScilabTokens() { }
151
152         void add(final int type, final int pos) {
153            tokenType.add(type);
154            tokenPos.add(pos);
155         }
156
157         public final List<Integer> getTokenType() {
158            return tokenType;
159         }
160
161         public final List<Integer> getTokenPos() {
162            return tokenPos;
163         }
164      }
165 %}
166
167 /* main character classes */
168 eol = \n
169
170 open = "[" | "(" | "{"
171 close = "]" | ")" | "}"
172
173 comment = "//"
174
175 quote = "'"
176
177 dquote = "\""
178
179 cstes = "%t" | "%T" | "%f" | "%F" | "%e" | "%pi" | "%inf" | "%i" | "%z" | "%s" | "%nan" | "%eps" | "SCI" | "WSCI" | "SCIHOME" | "TMPDIR"
180
181 operator = ".'" | ".*" | "./" | ".\\" | ".^" | ".**" | "+" | "-" | "/" | "\\" | "*" | "^" | "**" | "==" | "~=" | "<>" | "<" | ">" | "<=" | ">=" | ".*." | "./." | ".\\." | "/." | "=" | "&" | "|" | "@" | "@=" | "~"
182
183 functionKwds = "function" | "endfunction"
184
185 structureKwds = "then" | "do" | "catch" | "case"
186
187 elseif = "elseif" | "else"
188
189 openCloseStructureKwds = "if" | "for" | "while" | "try" | "select" | "end"
190
191 controlKwds = "abort" | "break" | "quit" | "return" | "resume" | "pause" | "continue" | "exit"
192
193 authors = "Calixte Denizet" | "Calixte DENIZET" | "Sylvestre Ledru" | "Sylvestre LEDRU" | "Yann Collette" | "Yann COLLETTE" | "Allan Cornet" | "Allan CORNET" | "Antoine Elias" | "Antoine ELIAS" | "Bruno Jofret" | "Bruno JOFRET" | "Claude Gomez" | "Claude GOMEZ" | "Clement David" | "Clement DAVID" | "Manuel Juliachs" | "Manuel JULIACHS" | "Michael Baudin" | "Michael BAUDIN" | "Pierre Lando" | "Pierre LANDO" | "Pierre Marechal" | "Pierre MARECHAL" | "Sheldon Cooper" | "Leonard Hofstadter" | "Serge Steer" | "Serge STEER" | "Vincent Couvert" | "Vincent COUVERT" | "Vincent Liard" | "Vincent LIARD" | "Adeline Carnis" | "Adeline CARNIS" | "Simon Gareste" | "Simon GARESTE" | "Cedric Delamarre" | "Cedric DELAMARRE" | "Inria" | "INRIA" | "DIGITEO" | "Digiteo" | "Scilab Enterprises" | "ENPC"
194
195 error = "Scilab Entreprises" | "Scilab Entreprise" | "Scilab Enterprise"
196
197 break = ".."(".")*
198 breakinstring = {break}[ \t]*({comment} | {eol})
199
200 special = "$" | ":" | {break}
201
202 string = (([^ \t\'\"\r\n\.]*)|([\'\"]{2}))+
203
204 id = ([a-zA-Z%_#!?][a-zA-Z0-9_#!$?]*)|("$"[a-zA-Z0-9_#!$?]+)
205
206 badid = ([0-9$][a-zA-Z0-9_#!$?]+)
207 whitabs = (" "+"\t" | "\t"+" ")[ \t]*
208 badop = [+-]([\*\/\\\^] | "."[\*\+\-\/\\\^]) | ":=" | "->" | " !=" | "&&" | "||" | ([*+-/\\\^]"=")
209
210 dot = "."
211
212 url = "http://"[^ \t\f\n\r\'\"]+
213 mail = "<"[ \t]*[a-zA-Z0-9_\.\-]+"@"([a-zA-Z0-9\-]+".")+[a-zA-Z]{2,5}[ \t]*">"
214
215 latex = "$"(([^$]*|"\\$")+)"$"
216 latexinstring = (\"|\')"$"(([^$\'\"]*|"\\$"|([\'\"]{2}))+)"$"(\"|\')
217
218 digit = [0-9]
219 exp = [dDeE][+-]?{digit}*
220 number = ({digit}+"."?{digit}*{exp}?)|("."{digit}+{exp}?)
221
222 %x QSTRING, COMMENT, FIELD, COMMANDS, COMMANDSWHITE, BREAKSTRING
223
224 %%
225
226 <YYINITIAL> {
227   {comment}                      {
228                                    transposable = false;
229                                    yypushback(2);
230                                    yybegin(COMMENT);
231                                  }
232
233   {operator}                     {
234                                    transposable = false;
235                                    return ScilabLexerConstants.OPERATOR;
236                                  }
237
238   {functionKwds}                 {
239                                    transposable = false;
240                                    return ScilabLexerConstants.FKEYWORD;
241                                  }
242
243   {openCloseStructureKwds}       {
244                                    transposable = false;
245                                    return ScilabLexerConstants.OSKEYWORD;
246                                  }
247
248   {structureKwds}                {
249                                    transposable = false;
250                                    return ScilabLexerConstants.SKEYWORD;
251                                  }
252
253   {elseif}                       {
254                                    transposable = false;
255                                    return ScilabLexerConstants.ELSEIF;
256                                  }
257
258   {controlKwds}                  {
259                                    transposable = false;
260                                    return ScilabLexerConstants.CKEYWORD;
261                                  }
262
263   {cstes}                        {
264                                    transposable = true;
265                                    return ScilabLexerConstants.CONSTANTES;
266                                  }
267
268   {id}                           {
269                                    transposable = true;
270                                    String str = yytext();
271                                    if (commands.contains(str)) {
272                                        yybegin(COMMANDS);
273                                        return ScilabLexerConstants.COMMANDS;
274                                    } else if (macros.contains(str)) {
275                                        yybegin(COMMANDS);
276                                        return ScilabLexerConstants.MACROS;
277                                    } else if (infile.contains(str)) {
278                                        yybegin(COMMANDS);
279                                        return ScilabLexerConstants.MACROINFILE;
280                                    } else {
281                                        List<String>[] arr = doc.getInOutArgs(start + yychar);
282                                        if (arr != null && (arr[0].contains(str) || arr[1].contains(str))) {
283                                            return ScilabLexerConstants.INPUTOUTPUTARGS;
284                                        } else if (variables.contains(str)) {
285                                            return ScilabLexerConstants.VARIABLES;
286                                        }
287                                    }
288                                    return ScilabLexerConstants.ID;
289                                  }
290
291   {number}                       {
292                                    transposable = true;
293                                    return ScilabLexerConstants.NUMBER;
294                                  }
295
296   {special}                      {
297                                    transposable = false;
298                                    return ScilabLexerConstants.SPECIAL;
299                                  }
300
301   {dot}                          {
302                                    transposable = false;
303                                    yybegin(FIELD);
304                                    return ScilabLexerConstants.OPERATOR;
305                                  }
306
307   {latexinstring}                {
308                                    return ScilabLexerConstants.LATEXINSTRING;
309                                  }
310
311   {quote}                        {
312                                     if (transposable) {
313                                        return ScilabLexerConstants.TRANSP;
314                                     } else {
315                                        beginString = zzStartRead;
316                                        yybegin(QSTRING);
317                                        return ScilabLexerConstants.STRING;
318                                     }
319                                  }
320
321   {open}                         {
322                                    transposable = false;
323                                    return ScilabLexerConstants.OPENCLOSE;
324                                  }
325
326   {close}                        {
327                                    transposable = true;
328                                    return ScilabLexerConstants.OPENCLOSE;
329                                  }
330
331   {dquote}                       {
332                                    transposable = false;
333                                    beginString = zzStartRead;
334                                    yybegin(QSTRING);
335                                    return ScilabLexerConstants.STRING;
336                                  }
337
338   {badid}                        |
339   {badop}                        |
340   {whitabs}                      {
341                                    return ScilabLexerConstants.ERROR;
342                                  }
343
344   " "                            {
345                                    transposable = false;
346                                    return ScilabLexerConstants.WHITE;
347                                  }
348
349   "\t"                           {
350                                    transposable = false;
351                                    return ScilabLexerConstants.TAB;
352                                  }
353
354   .                              |
355   {eol}                          {
356                                    transposable = false;
357                                    return ScilabLexerConstants.DEFAULT;
358                                  }
359 }
360
361 <COMMANDS> {
362   [ \t]*"("                      {
363                                    yypushback(yylength());
364                                    yybegin(YYINITIAL);
365                                  }
366
367   " "                            {
368                                    yybegin(COMMANDSWHITE);
369                                    return ScilabLexerConstants.WHITE;
370                                  }
371
372   "\t"                           {
373                                    yybegin(COMMANDSWHITE);
374                                    return ScilabLexerConstants.TAB;
375                                  }
376   .
377                                  {
378                                    yypushback(1);
379                                    yybegin(YYINITIAL);
380                                  }
381
382   {eol}                          { }
383 }
384
385 <COMMANDSWHITE> {
386   {comment}                      {
387                                    transposable = false;
388                                    yypushback(2);
389                                    yybegin(COMMENT);
390                                  }
391
392   ([^ \t,;/]*) | ("/"[^ /]*)     {
393                                    return ScilabLexerConstants.STRING;
394                                  }
395
396   " "                            {
397                                    return ScilabLexerConstants.WHITE;
398                                  }
399
400   "\t"                           {
401                                    return ScilabLexerConstants.TAB;
402                                  }
403
404   .                              |
405   {eol}                          {
406                                    yypushback(1);
407                                    yybegin(YYINITIAL);
408                                  }
409 }
410
411 <FIELD> {
412   {id}                           {
413                                    transposable = true;
414                                    return ScilabLexerConstants.FIELD;
415                                  }
416
417   .                              |
418   {eol}                          {
419                                    yypushback(1);
420                                    yybegin(YYINITIAL);
421                                  }
422 }
423
424 <QSTRING> {
425   {breakinstring}                {
426                                    yypushback(yylength());
427                                    yybegin(BREAKSTRING);
428                                    transposable = false;
429                                    return ScilabLexerConstants.STRING;
430                                  }
431
432   " "                            {
433                                    return ScilabLexerConstants.WHITE_STRING;
434                                  }
435
436   "\t"                           {
437                                    return ScilabLexerConstants.TAB_STRING;
438                                  }
439
440   {string}                       |
441   "."                            {
442                                    return ScilabLexerConstants.STRING;
443                                  }
444
445   (\'|\")                        {
446                                    transposable = false;
447                                    yybegin(YYINITIAL);
448                                    return ScilabLexerConstants.STRING;
449                                  }
450
451   .                              |
452   {eol}                          {
453                                    return ScilabLexerConstants.DEFAULT;
454                                  }
455 }
456
457 <COMMENT> {
458   {error}                        {
459                                    return ScilabLexerConstants.ERROR;
460                                  }
461
462
463   {authors}                      {
464                                    return ScilabLexerConstants.AUTHORS;
465                                  }
466
467   {url}                          {
468                                    return ScilabLexerConstants.URL;
469                                  }
470
471   {mail}                         {
472                                    return ScilabLexerConstants.MAIL;
473                                  }
474
475   {latex}                        {
476                                    return ScilabLexerConstants.LATEX;
477                                  }
478
479   " "                            {
480                                    return ScilabLexerConstants.WHITE_COMMENT;
481                                  }
482
483   "\t"                           {
484                                    return ScilabLexerConstants.TAB_COMMENT;
485                                  }
486
487   .                              {
488                                    return ScilabLexerConstants.COMMENT;
489                                  }
490
491   {eol}                          {
492                                    yybegin(YYINITIAL);
493                                    return ScilabLexerConstants.DEFAULT;
494                                  }
495 }
496
497 <BREAKSTRING> {
498   {break}                        {
499                                    breakstring = true;
500                                    return ScilabLexerConstants.SPECIAL;
501                                  }
502
503   " "                            {
504                                    return ScilabLexerConstants.WHITE;
505                                  }
506
507   "\t"                           {
508                                    return ScilabLexerConstants.TAB;
509                                  }
510
511   {comment}                      {
512                                    transposable = false;
513                                    yypushback(2);
514                                    yybegin(COMMENT);
515                                  }
516
517   .                              |
518   {eol}                          {
519                                    return ScilabLexerConstants.DEFAULT;
520                                  }
521 }
522
523 <<EOF>>                          {
524                                    return ScilabLexerConstants.EOF;
525                                  }