From e3e49fd882f89554a790b76cbf1b350b8d1f55b2 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Cl=C3=A9ment=20DAVID?= Date: Wed, 18 Jan 2017 22:34:49 +0100 Subject: [PATCH] Xcos search: index help JARs Change-Id: I9ba29bc09362ffe6965e5182920b8c0bb8ae04ea --- .../modules/xcos/palette/PaletteIndexer.java | 211 +++++++++++++------- .../modules/xcos/palette/PaletteSearchManager.java | 59 ++---- .../modules/xcos/palette/PaletteSearcher.java | 46 ++--- .../modules/xcos/palette/view/PaletteView.java | 6 +- 4 files changed, 185 insertions(+), 137 deletions(-) diff --git a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteIndexer.java b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteIndexer.java index cf9920e..d31a0a7 100644 --- a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteIndexer.java +++ b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteIndexer.java @@ -1,6 +1,7 @@ /* * Scilab ( http://www.scilab.org/ ) - This file is part of Scilab * Copyright (C) 2015 - Marcos CARDINOT + * Copyright (C) 2017 - Scilab Enterprises - Clement DAVID * * This file must be used under the terms of the CeCILL. * This source file is licensed as described in the file COPYING, which @@ -9,25 +10,29 @@ * http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.txt * */ - package org.scilab.modules.xcos.palette; import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; -import java.io.FilenameFilter; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Hashtable; import java.util.List; -import java.util.Set; -import java.util.Map.Entry; - +import java.util.Map; +import java.util.Optional; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.lucene.document.Document; + import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; @@ -37,99 +42,165 @@ import org.scilab.modules.xcos.palette.model.PaletteBlock; /** * Index the help pages of all blocks. + * * @author Marcos Cardinot */ public final class PaletteIndexer { private PaletteSearchManager mgr; - private List roots; + private List roots; /** * Default constructor + * * @param psm PaletteSearchManager */ public PaletteIndexer(PaletteSearchManager psm) { mgr = psm; // javaHelp directories - roots = new ArrayList(); - roots.add(ScilabConstants.SCI.getAbsolutePath() + "/modules/helptools/javaHelp"); + roots = new ArrayList<>(); + + // for local builds + roots.add(new File(ScilabConstants.SCI.getAbsolutePath() + "/modules/helptools/javaHelp")); + + // for binary version + roots.add(new File(ScilabConstants.SCI.getAbsolutePath() + "/modules/helptools/jar")); } - /** - * @param ht Hashtable - */ - public void createIndex(Hashtable> ht) { + public void createIndex(Map blockNameToPalette) { try { mgr.getIndexWriter().deleteAll(); - Set< Entry> > treePaths = ht.entrySet(); - for (Entry> entry : treePaths) { - String treePath = entry.getKey(); - List blocks = entry.getValue(); - for (PaletteBlock block : blocks) { - indexBlock(treePath, block.getName()); + + // insert all block names + for (String blk : blockNameToPalette.keySet()) { + index(blk); + } + + // insert all help pages + for (File r : roots) { + if (!r.exists()) { + continue; } + + Files.walkFileTree(r.toPath(), new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + String fname = file.getFileName().toString(); + int dot = fname.lastIndexOf('.'); + if (dot <= 0) { + return FileVisitResult.CONTINUE; + } + + String basename = fname.substring(0, dot); + if (fname.endsWith(".jar")) { + try (JarFile jf = new JarFile(file.toFile())) { + index(file.toAbsolutePath().toUri().toString(), basename, jf, blockNameToPalette); + } + } else if (fname.endsWith(".html")) { + // this is a regular file + if (blockNameToPalette.containsKey(basename)) { + index(basename, file.toUri().toURL()); + } + } + + return FileVisitResult.CONTINUE; + } + }); } + mgr.getIndexWriter().commit(); - } catch (IOException e) { - e.printStackTrace(); + } catch (IOException ex) { + Logger.getLogger(PaletteIndexer.class.getName()).log(Level.SEVERE, null, ex); } } - /** - * @param treePath tree path - * @param blockName block name - * @throws IOException If there is a low-level I/O error - */ - private void indexBlock(String treePath, String blockName) throws IOException { - Document doc = new Document(); - doc.add(new StringField("blockName", blockName, Field.Store.YES)); - doc.add(new StringField("treePath", treePath, Field.Store.YES)); - - List helpPages = findHelpPages(blockName); - if (helpPages.isEmpty()) { - doc.add(new TextField("helpPage", blockName, Field.Store.YES)); - } else { - for (File helpPage : helpPages) { - InputStream stream = new FileInputStream(helpPage); - doc.add(new StringField("filePath", helpPage.getAbsolutePath(), Field.Store.YES)); - doc.add(new TextField("helpPage", new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)))); + private void index(String fileURI, String basename, JarFile jf, Map blockNameToTreePath) { + class Entry { + + Entry(JarEntry jarEntry) { + je = jarEntry; } - } - mgr.getIndexWriter().addDocument(doc); - } + JarEntry je; + String blk; + URL url; - /** - * @param blockName block name - * @return File[] - */ - private List findHelpPages(final String blockName) { - List subdirs = new ArrayList(); - for (String root : roots) { - File r = new File(root); - if (!r.exists()) { - continue; + boolean block() { + int slash = je.getName().lastIndexOf('/') + 1; + int dot = je.getName().lastIndexOf('.'); + if (slash < 0 || dot < 0) { + return false; + } + + blk = je.getName().substring(slash, dot); + return true; } - String[] ss = r.list(new FilenameFilter() { - @Override - public boolean accept(File current, String name) { - return new File(current, name).isDirectory(); + boolean url() { + try { + // we have to generate a url in the form before indexing + // jar:file:/modules/helptools/jar/scilab_en_US_help.jar!/scilab_en_US_help/blockName.html + url = new URL("jar:" + fileURI + "!/" + basename + '/' + blk + ".html"); + } catch (MalformedURLException ex) { + Logger.getLogger(PaletteIndexer.class.getName()).log(Level.SEVERE, null, ex); } - }); - for (int i = 0; i < ss.length; ++i) { - ss[i] = root + File.separator + ss[i]; + return true; } - subdirs.addAll(Arrays.asList(ss)); } - List helpPages = new ArrayList(); - for (String dir : subdirs) { - File file = new File(dir + File.separator + blockName + ".html"); - if (file.exists()) { - helpPages.add(file); + jf.stream() + .map(je -> new Entry(je)) + .filter(e -> e.block()) + .filter(e -> blockNameToTreePath.containsKey(e.blk)) + .filter(e -> e.url()) + .forEach(e -> index(e.blk, e.url)); + } + + private void index(String basename, URL url) { + try { + Document doc = new Document(); + + // add the block name + Field refname = new TextField("refname", basename, Field.Store.YES); + refname.setBoost(100f); + doc.add(refname); + + // add the refpurpose + try (BufferedReader r = new BufferedReader(new InputStreamReader(url.openStream()))) { + Optional found = r.lines().filter(l -> l.contains("refpurpose")).findFirst(); + + Field refpurpose; + if (found.isPresent()) { + refpurpose = new TextField("refpurpose", found.get(), Field.Store.YES); + } else { + refpurpose = new TextField("refpurpose", "", Field.Store.YES); + } + + refpurpose.setBoost(10f); + doc.add(refpurpose); } + + // add the html content + try (BufferedReader r = new BufferedReader(new InputStreamReader(url.openStream()))) { + doc.add(new TextField("content", r)); + } + + mgr.getIndexWriter().addDocument(doc); + } catch (IOException e) { + Logger.getLogger(PaletteIndexer.class.getName()).log(Level.SEVERE, null, e); + } + } + + private void index(String block) { + try { + Document doc = new Document(); + doc.add(new StringField("refname", block, Field.Store.YES)); + doc.add(new StringField("refpurpose", block, Field.Store.YES)); + doc.add(new TextField("content", block, Field.Store.YES)); + + mgr.getIndexWriter().addDocument(doc); + } catch (IOException e) { + Logger.getLogger(PaletteIndexer.class.getName()).log(Level.SEVERE, null, e); } - return helpPages; } } diff --git a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearchManager.java b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearchManager.java index a980992..2efc1ac 100644 --- a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearchManager.java +++ b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearchManager.java @@ -1,6 +1,7 @@ /* * Scilab ( http://www.scilab.org/ ) - This file is part of Scilab * Copyright (C) 2015 - Marcos CARDINOT + * Copyright (C) 2017 - Scilab Enterprises - Clement DAVID * * This file must be used under the terms of the CeCILL. * This source file is licensed as described in the file COPYING, which @@ -14,22 +15,23 @@ package org.scilab.modules.xcos.palette; import java.io.File; import java.io.IOException; -import java.util.Hashtable; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.JScrollPane; import javax.swing.tree.TreeModel; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; -import org.scilab.modules.xcos.palette.PaletteIndexer; -import org.scilab.modules.xcos.palette.PaletteSearcher; -import org.scilab.modules.xcos.palette.model.PaletteBlock; import org.scilab.modules.xcos.palette.model.PaletteBlock; import org.scilab.modules.xcos.palette.model.PaletteNode; import org.scilab.modules.xcos.palette.model.PreLoaded; @@ -51,7 +53,7 @@ public final class PaletteSearchManager { private PaletteIndexer paletteIndexer; private PaletteSearcher paletteSearcher; private PaletteSearchView view; - private Hashtable> ht; + private Map nameToPalette; /** Default constructor */ public PaletteSearchManager() { @@ -61,14 +63,14 @@ public final class PaletteSearchManager { IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setOpenMode(OpenMode.CREATE); writer = new IndexWriter(ramDir, config); - } catch (IOException e) { - e.printStackTrace(); + } catch (IOException ex) { + Logger.getLogger(PaletteSearchManager.class.getName()).log(Level.SEVERE, null, ex); } + nameToPalette = new HashMap<>(); view = new PaletteSearchView(); paletteIndexer = new PaletteIndexer(this); paletteSearcher = new PaletteSearcher(this); - ht = new Hashtable>(); } /** @@ -91,14 +93,14 @@ public final class PaletteSearchManager { indexIsOutdated = false; } - List blockPaths = paletteSearcher.search(query); - for (String blockPath : blockPaths) { - PaletteBlock block = getBlock(blockPath); + List found = paletteSearcher.search(query); + for (Document doc : found) { + PaletteBlock block = nameToPalette.get(doc.get("refname")); if (block != null) { view.addBlock(block); } } - view.setText(queryLabel + blockPaths.size() + " " + XcosMessages.MATCHES); + view.setText(queryLabel + found.size() + " " + XcosMessages.MATCHES); view.revalidate(); scrollPane.revalidate(); } @@ -110,7 +112,7 @@ public final class PaletteSearchManager { TreeModel model = PaletteManagerView.get().getTree().getModel(); if (model != null) { loadHashTable(model, model.getRoot(), ""); - paletteIndexer.createIndex(ht); + paletteIndexer.createIndex(nameToPalette); } } @@ -125,12 +127,11 @@ public final class PaletteSearchManager { for (int i = 0; i < cc; ++i) { PaletteNode node = (PaletteNode) model.getChild(obj, i); if (node instanceof PreLoaded) { - ht.put(treePath + File.separator + node.getName(), - ((PreLoaded) node).getBlock()); + for (PaletteBlock b : ((PreLoaded) node).getBlock()) { + nameToPalette.put(b.getName(), b); + } } else { - loadHashTable(model, - node, - treePath + File.separator + node.toString()); + loadHashTable(model, node, treePath + File.separator + node.toString()); } } } @@ -159,26 +160,4 @@ public final class PaletteSearchManager { public IndexWriter getIndexWriter() { return writer; } - - /** - * Get block - * @param blockPath the block path - * @return PaletteBlock - */ - public PaletteBlock getBlock(String blockPath) { - int lastSeparator = blockPath.lastIndexOf(File.separator); - String treePath = blockPath.substring(0, lastSeparator); - String blockName = blockPath.substring(lastSeparator + 1); - List blocks = ht.get(treePath); - if (blocks == null) { - return null; - } - - for (PaletteBlock block : blocks) { - if (blockName.equals(block.getName())) { - return block; - } - } - return null; - } } diff --git a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearcher.java b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearcher.java index ca68aa1..7697cd7 100644 --- a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearcher.java +++ b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearcher.java @@ -1,6 +1,7 @@ /* * Scilab ( http://www.scilab.org/ ) - This file is part of Scilab * Copyright (C) 2015 - Marcos CARDINOT + * Copyright (C) 2017 - Scilab Enterprises - Clement DAVID * * This file must be used under the terms of the CeCILL. * This source file is licensed as described in the file COPYING, which @@ -12,16 +13,17 @@ package org.scilab.modules.xcos.palette; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; @@ -35,53 +37,47 @@ import org.scilab.modules.xcos.utils.XcosConstants; public final class PaletteSearcher { private PaletteSearchManager mgr; - private QueryParser parser; /** - * Default constructor + * Default constructor * @param psm PaletteSearchManager */ public PaletteSearcher(PaletteSearchManager psm) { mgr = psm; - parser = new QueryParser("helpPage", mgr.getAnalyzer()); - parser.setAllowLeadingWildcard(true); } /** * @param str Query * @return paths to the found blocks */ - public List search(String str) { - List blockPaths = new ArrayList(); - try { - IndexReader reader = DirectoryReader.open(mgr.getIndexWriter(), true); + public List search(String str) { + List found = new ArrayList<>(); + try (IndexReader reader = DirectoryReader.open(mgr.getIndexWriter(), true)) { IndexSearcher searcher = new IndexSearcher(reader); - // escape special characters -- except * - String escaped = QueryParser.escape(str); - escaped = escaped.replaceAll("\\\\\\*", "*"); + StandardQueryParser queryParserHelper = new StandardQueryParser(); + queryParserHelper.setAllowLeadingWildcard(true); + queryParserHelper.setLowercaseExpandedTerms(true); + queryParserHelper.setAnalyzer(mgr.getAnalyzer()); + queryParserHelper.setMultiFields(new String[] {"refname", "refpurpose", "content"}); - Query query = parser.parse(escaped); + Query query = queryParserHelper.parse(str, null); TopDocs results = searcher.search(query, XcosConstants.MAX_HITS); ScoreDoc[] hits = results.scoreDocs; - if (hits.length == 0 && !str.contains("*")) { - escaped = "*" + escaped + "*"; - query = parser.parse(escaped); + if (hits.length == 0) { + query = queryParserHelper.parse("*" + str + "*", null); results = searcher.search(query, XcosConstants.MAX_HITS); hits = results.scoreDocs; } for (int i = 0; i < hits.length; i++) { Document doc = searcher.doc(hits[i].doc); - blockPaths.add(doc.get("treePath") + File.separator + doc.get("blockName")); + found.add(doc); } - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } catch (ParseException e) { - e.printStackTrace(); + } catch (IOException | QueryNodeException e) { + Logger.getLogger(PaletteSearcher.class.getName()).log(Level.SEVERE, null, e); } - return blockPaths; + return found; } } diff --git a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/view/PaletteView.java b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/view/PaletteView.java index 117fa9e..0b3d502 100644 --- a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/view/PaletteView.java +++ b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/view/PaletteView.java @@ -1,7 +1,7 @@ /* * Scilab ( http://www.scilab.org/ ) - This file is part of Scilab * Copyright (C) 2010 - DIGITEO - Clement DAVID - * Copyright (C) 2011-2015 - Scilab Enterprises - Clement DAVID + * Copyright (C) 2011-2017 - Scilab Enterprises - Clement DAVID * Copyright (C) 2015 - Marcos CARDINOT * * Copyright (C) 2012 - 2016 - Scilab Enterprises @@ -74,7 +74,9 @@ public class PaletteView extends JPanel implements Scrollable { @Override public void focusGained(FocusEvent e) { try { - ((PaletteBlockView) getComponent(0)).getController().setSelected(true); + if (getComponentCount() > 0) { + ((PaletteBlockView) getComponent(0)).getController().setSelected(true); + } } catch (ClassCastException err) { } catch (NullPointerException err) { } -- 1.7.9.5