Xcos search: index help JARs
[scilab.git] / scilab / modules / xcos / src / java / org / scilab / modules / xcos / palette / PaletteIndexer.java
1 /*
2  * Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
3  * Copyright (C) 2015 - Marcos CARDINOT
4  * Copyright (C) 2017 - Scilab Enterprises - Clement DAVID
5  *
6  * This file must be used under the terms of the CeCILL.
7  * This source file is licensed as described in the file COPYING, which
8  * you should have received as part of this distribution.  The terms
9  * are also available at
10  * http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.txt
11  *
12  */
13 package org.scilab.modules.xcos.palette;
14
15 import java.io.BufferedReader;
16 import java.io.File;
17 import java.io.IOException;
18 import java.io.InputStreamReader;
19 import java.net.MalformedURLException;
20 import java.net.URL;
21 import java.nio.file.FileVisitResult;
22 import java.nio.file.Files;
23 import java.nio.file.Path;
24 import java.nio.file.SimpleFileVisitor;
25 import java.nio.file.attribute.BasicFileAttributes;
26 import java.util.ArrayList;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Optional;
30 import java.util.jar.JarEntry;
31 import java.util.jar.JarFile;
32 import java.util.logging.Level;
33 import java.util.logging.Logger;
34 import org.apache.lucene.document.Document;
35
36 import org.apache.lucene.document.Field;
37 import org.apache.lucene.document.StringField;
38 import org.apache.lucene.document.TextField;
39
40 import org.scilab.modules.commons.ScilabConstants;
41 import org.scilab.modules.xcos.palette.model.PaletteBlock;
42
43 /**
44  * Index the help pages of all blocks.
45  *
46  * @author Marcos Cardinot <mcardinot@gmail.com>
47  */
48 public final class PaletteIndexer {
49
50     private PaletteSearchManager mgr;
51     private List<File> roots;
52
53     /**
54      * Default constructor
55      *
56      * @param psm PaletteSearchManager
57      */
58     public PaletteIndexer(PaletteSearchManager psm) {
59         mgr = psm;
60         // javaHelp directories
61         roots = new ArrayList<>();
62
63         // for local builds
64         roots.add(new File(ScilabConstants.SCI.getAbsolutePath() + "/modules/helptools/javaHelp"));
65
66         // for binary version
67         roots.add(new File(ScilabConstants.SCI.getAbsolutePath() + "/modules/helptools/jar"));
68     }
69
70     public void createIndex(Map<String, PaletteBlock> blockNameToPalette) {
71         try {
72             mgr.getIndexWriter().deleteAll();
73
74             // insert all block names
75             for (String blk : blockNameToPalette.keySet()) {
76                 index(blk);
77             }
78
79             // insert all help pages
80             for (File r : roots) {
81                 if (!r.exists()) {
82                     continue;
83                 }
84
85                 Files.walkFileTree(r.toPath(), new SimpleFileVisitor<Path>() {
86                     @Override
87                     public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
88                         String fname = file.getFileName().toString();
89                         int dot = fname.lastIndexOf('.');
90                         if (dot <= 0) {
91                             return FileVisitResult.CONTINUE;
92                         }
93
94                         String basename = fname.substring(0, dot);
95                         if (fname.endsWith(".jar")) {
96                             try (JarFile jf = new JarFile(file.toFile())) {
97                                 index(file.toAbsolutePath().toUri().toString(), basename, jf, blockNameToPalette);
98                             }
99                         } else if (fname.endsWith(".html")) {
100                             // this is a regular file
101                             if (blockNameToPalette.containsKey(basename)) {
102                                 index(basename, file.toUri().toURL());
103                             }
104                         }
105
106                         return FileVisitResult.CONTINUE;
107                     }
108                 });
109             }
110
111             mgr.getIndexWriter().commit();
112         } catch (IOException ex) {
113             Logger.getLogger(PaletteIndexer.class.getName()).log(Level.SEVERE, null, ex);
114         }
115     }
116
117     private void index(String fileURI, String basename, JarFile jf, Map<String, PaletteBlock> blockNameToTreePath) {
118         class Entry {
119
120             Entry(JarEntry jarEntry) {
121                 je = jarEntry;
122             }
123
124             JarEntry je;
125             String blk;
126             URL url;
127
128             boolean block() {
129                 int slash = je.getName().lastIndexOf('/') + 1;
130                 int dot = je.getName().lastIndexOf('.');
131                 if (slash < 0 || dot < 0) {
132                     return false;
133                 }
134
135                 blk = je.getName().substring(slash, dot);
136                 return true;
137             }
138
139             boolean url() {
140                 try {
141                     // we have to generate a url in the form before indexing
142                     // jar:file:/modules/helptools/jar/scilab_en_US_help.jar!/scilab_en_US_help/blockName.html
143                     url = new URL("jar:" + fileURI + "!/" + basename + '/' + blk + ".html");
144                 } catch (MalformedURLException ex) {
145                     Logger.getLogger(PaletteIndexer.class.getName()).log(Level.SEVERE, null, ex);
146                 }
147                 return true;
148             }
149         }
150
151         jf.stream()
152         .map(je -> new Entry(je))
153         .filter(e -> e.block())
154         .filter(e -> blockNameToTreePath.containsKey(e.blk))
155         .filter(e -> e.url())
156         .forEach(e -> index(e.blk, e.url));
157     }
158
159     private void index(String basename, URL url) {
160         try {
161             Document doc = new Document();
162
163             // add the block name
164             Field refname = new TextField("refname", basename, Field.Store.YES);
165             refname.setBoost(100f);
166             doc.add(refname);
167
168             // add the refpurpose
169             try (BufferedReader r = new BufferedReader(new InputStreamReader(url.openStream()))) {
170                 Optional<String> found = r.lines().filter(l -> l.contains("refpurpose")).findFirst();
171
172                 Field refpurpose;
173                 if (found.isPresent()) {
174                     refpurpose = new TextField("refpurpose", found.get(), Field.Store.YES);
175                 } else {
176                     refpurpose = new TextField("refpurpose", "", Field.Store.YES);
177                 }
178
179                 refpurpose.setBoost(10f);
180                 doc.add(refpurpose);
181             }
182
183             // add the html content
184             try (BufferedReader r = new BufferedReader(new InputStreamReader(url.openStream()))) {
185                 doc.add(new TextField("content", r));
186             }
187
188             mgr.getIndexWriter().addDocument(doc);
189         } catch (IOException e) {
190             Logger.getLogger(PaletteIndexer.class.getName()).log(Level.SEVERE, null, e);
191         }
192     }
193
194     private void index(String block) {
195         try {
196             Document doc = new Document();
197             doc.add(new StringField("refname", block, Field.Store.YES));
198             doc.add(new StringField("refpurpose", block, Field.Store.YES));
199             doc.add(new TextField("content", block, Field.Store.YES));
200
201             mgr.getIndexWriter().addDocument(doc);
202         } catch (IOException e) {
203             Logger.getLogger(PaletteIndexer.class.getName()).log(Level.SEVERE, null, e);
204         }
205     }
206 }