Incremental update issue in lucene

Question

I am creating a program that can index many text files in another folder. so each folder with text files is indexed, and its index is stored in a different folder. therefore, this other folder acts as a universal index of all the files on my computer. and I use lucene to achieve this because lucene fully supports incremental updates. this is the source code in which i use it for indexing.

public class SimpleFileIndexer {


public static void main(String[] args) throws Exception   {

    int i=0;
    while(i<2) {
    File indexDir = new File("C:/Users/Raden/Documents/myindex");
    File dataDir = new File("C:/Users/Raden/Documents/indexthis");
    String suffix = "txt";

    SimpleFileIndexer indexer = new SimpleFileIndexer();

    int numIndex = indexer.index(indexDir, dataDir, suffix);

    System.out.println("Total files indexed " + numIndex);
    i++;
    Thread.sleep(1000);

    }
}


private int index(File indexDir, File dataDir, String suffix) throws Exception {
    RAMDirectory ramDir = new RAMDirectory();          // 1
    @SuppressWarnings("deprecation")
    IndexWriter indexWriter = new IndexWriter(
            ramDir,                                    // 2
            new StandardAnalyzer(Version.LUCENE_CURRENT),
            true,
            IndexWriter.MaxFieldLength.UNLIMITED);
    indexWriter.setUseCompoundFile(false);
    indexDirectory(indexWriter, dataDir, suffix);
    int numIndexed = indexWriter.maxDoc();
    indexWriter.optimize();
    indexWriter.close();

    Directory.copy(ramDir, FSDirectory.open(indexDir), false); // 3

    return numIndexed;
}


private void indexDirectory(IndexWriter indexWriter, File dataDir, String suffix)  throws IOException {
    File[] files = dataDir.listFiles();
    for (int i = 0; i < files.length; i++) {
        File f = files[i];
        if (f.isDirectory()) {
            indexDirectory(indexWriter, f, suffix);
        }
        else {
            indexFileWithIndexWriter(indexWriter, f, suffix);
        }
    }
}

private void indexFileWithIndexWriter(IndexWriter indexWriter, File f, String suffix) throws IOException {
    if (f.isHidden() || f.isDirectory() || !f.canRead() || !f.exists()) {
        return;
    }
    if (suffix!=null && !f.getName().endsWith(suffix)) {
        return;
    }
    System.out.println("Indexing file " + f.getCanonicalPath());

    Document doc = new Document();
    doc.add(new Field("contents", new FileReader(f)));      
doc.add(new Field("filename", f.getCanonicalPath(), Field.Store.YES, Field.Index.ANALYZED));
    indexWriter.addDocument(doc);
} }

and this is the source code that I use to search for the index created by lucene

public class SimpleSearcher {

public static void main(String[] args) throws Exception {

    File indexDir = new File("C:/Users/Raden/Documents/myindex");
    String query = "revolution";
    int hits = 100;

    SimpleSearcher searcher = new SimpleSearcher();
    searcher.searchIndex(indexDir, query, hits);

}

private void searchIndex(File indexDir, String queryStr, int maxHits) throws Exception {

    Directory directory = FSDirectory.open(indexDir);

    IndexSearcher searcher = new IndexSearcher(directory);
    @SuppressWarnings("deprecation")
    QueryParser parser = new QueryParser(Version.LUCENE_30, "contents", new StandardAnalyzer(Version.LUCENE_CURRENT));
    Query query = parser.parse(queryStr);

    TopDocs topDocs = searcher.search(query, maxHits);

    ScoreDoc[] hits = topDocs.scoreDocs;
    for (int i = 0; i < hits.length; i++) {
        int docId = hits[i].doc;
        Document d = searcher.doc(docId);
        System.out.println(d.get("filename"));
    }

    System.out.println("Found " + hits.length);

}

}

, , , , , . , , , , , , , , , , ? . , , , , , .

,

Answer 1

Directory.copy() , IndexWriter.addIndexes() .

. RAMDirectory , (. IndexWriter docs).

: Directory.copy() ramDir indexDir .addIndexes indexDir ramDir. .addIndexesNoOptimize ramDir ( ) .

, , RAMDir indexDir . .

private int index(File indexDir, File dataDir, String suffix) throws Exception {
    RAMDirectory ramDir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(ramDir,
        new StandardAnalyzer(Version.LUCENE_CURRENT), true,  
        IndexWriter.MaxFieldLength.UNLIMITED);
    indexWriter.setUseCompoundFile(false);
    indexDirectory(indexWriter, dataDir, suffix);
    int numIndexed = indexWriter.maxDoc();
    indexWriter.optimize();
    indexWriter.close();


    IndexWriter index = new IndexWriter(FSDirectory.open(indexDir),
        new StandardAnalyzer(Version.LUCENE_CURRENT), true,  
        IndexWriter.MaxFieldLength.UNLIMITED);
    index.addIndexesNoOptimize(ramDir);
    index.optimize();
    index.close();

    return numIndexed;
}

:

private int index(File indexDir, File dataDir, String suffix) throws Exception {

    IndexWriter index = new IndexWriter(FSDirectory.open(indexDir),
        new StandardAnalyzer(Version.LUCENE_CURRENT), true,  
        IndexWriter.MaxFieldLength.UNLIMITED);

    // tweak the settings for your hardware
    index.setUseCompoundFile(false);
    index.setRAMBufferSizeMB(256);
    index.setMergeFactor(30);

    indexDirectory(index, dataDir, suffix);

    index.optimize();
    int numIndexed = index.maxDoc();
    index.close();

    // you'll need to update indexDirectory() to keep track of indexed files
    return numIndexed;
}