001/*
002    Licensed to the Apache Software Foundation (ASF) under one
003    or more contributor license agreements.  See the NOTICE file
004    distributed with this work for additional information
005    regarding copyright ownership.  The ASF licenses this file
006    to you under the Apache License, Version 2.0 (the
007    "License"); you may not use this file except in compliance
008    with the License.  You may obtain a copy of the License at
009
010       http://www.apache.org/licenses/LICENSE-2.0
011
012    Unless required by applicable law or agreed to in writing,
013    software distributed under the License is distributed on an
014    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015    KIND, either express or implied.  See the License for the
016    specific language governing permissions and limitations
017    under the License.    
018 */
019package org.apache.wiki.search.tika;
020
021import org.apache.logging.log4j.LogManager;
022import org.apache.logging.log4j.Logger;
023import org.apache.tika.exception.TikaException;
024import org.apache.tika.metadata.ClimateForcast;
025import org.apache.tika.metadata.CreativeCommons;
026import org.apache.tika.metadata.Database;
027import org.apache.tika.metadata.HttpHeaders;
028import org.apache.tika.metadata.IPTC;
029import org.apache.tika.metadata.Metadata;
030import org.apache.tika.metadata.Office;
031import org.apache.tika.metadata.OfficeOpenXMLCore;
032import org.apache.tika.metadata.PDF;
033import org.apache.tika.metadata.TikaCoreProperties;
034import org.apache.tika.parser.AutoDetectParser;
035import org.apache.tika.sax.BodyContentHandler;
036import org.apache.wiki.api.core.Attachment;
037import org.apache.wiki.api.exceptions.ProviderException;
038import org.apache.wiki.attachment.AttachmentManager;
039import org.apache.wiki.search.LuceneSearchProvider;
040import org.xml.sax.ContentHandler;
041import org.xml.sax.SAXException;
042
043import java.io.IOException;
044import java.io.InputStream;
045import java.util.HashSet;
046import java.util.Set;
047
048/**
049 * Search provider that extends {link LuceneSearchProvider} using Apache Tika for indexing attachment content.
050 *
051 * @since 2.11.0
052 * @see <a href="https://issues.apache.org/jira/browse/JSPWIKI-469">JSPWIKI-469</a>
053 */
054public class TikaSearchProvider extends LuceneSearchProvider {
055
056    private static final Logger LOG = LogManager.getLogger( TikaSearchProvider.class );
057    final AutoDetectParser parser;
058    final Set< String > textualMetadataFields;
059
060    public TikaSearchProvider() {
061        parser = new AutoDetectParser();
062
063        // metadata fields that also are indexed
064        textualMetadataFields = new HashSet<>();
065        textualMetadataFields.add( TikaCoreProperties.TITLE.getName() );
066        textualMetadataFields.add( TikaCoreProperties.COMMENTS.getName() );
067        textualMetadataFields.add( TikaCoreProperties.SUBJECT.getName() );
068        textualMetadataFields.add( TikaCoreProperties.DESCRIPTION.getName() );
069        textualMetadataFields.add( TikaCoreProperties.TYPE.getName() );
070        textualMetadataFields.add( TikaCoreProperties.RESOURCE_NAME_KEY );
071        textualMetadataFields.add( PDF.DOC_INFO_TITLE.getName() );
072        textualMetadataFields.add( PDF.DOC_INFO_KEY_WORDS.getName() );
073        textualMetadataFields.add( PDF.DOC_INFO_SUBJECT.getName() );
074        textualMetadataFields.add( OfficeOpenXMLCore.SUBJECT.getName() );
075        textualMetadataFields.add( Office.KEYWORDS.getName() );
076        textualMetadataFields.add( TikaCoreProperties.TYPE.getName() );
077        textualMetadataFields.add( HttpHeaders.CONTENT_TYPE );
078        textualMetadataFields.add( IPTC.HEADLINE.getName() );
079        textualMetadataFields.add( Database.COLUMN_NAME.getName() );
080        textualMetadataFields.add( Database.TABLE_NAME.getName() );
081        textualMetadataFields.add( CreativeCommons.WORK_TYPE );
082        textualMetadataFields.add( ClimateForcast.COMMENT );
083        textualMetadataFields.add( ClimateForcast.HISTORY );
084        textualMetadataFields.add( ClimateForcast.INSTITUTION );
085    }
086
087    /**
088     * {@inheritDoc}
089     *
090     * @param att Attachment to get content for. Filename extension is used to determine the type of the attachment.
091     * @return String representing the content of the file.
092     */
093    @Override
094    protected String getAttachmentContent( final Attachment att ) {
095        final AttachmentManager mgr = getEngine().getManager( AttachmentManager.class );
096        final StringBuilder out = new StringBuilder();
097
098        try( final InputStream attStream = mgr.getAttachmentStream( att ) ) {
099            final Metadata metadata = new Metadata();
100            metadata.set( TikaCoreProperties.RESOURCE_NAME_KEY, att.getFileName() );
101
102            final ContentHandler handler = new BodyContentHandler(-1 );
103            // -1 disables the character size limit; otherwise only the first 100.000 characters are indexed
104
105            parser.parse( attStream, handler, metadata );
106            out.append( handler );
107
108            final String[] names = metadata.names();
109            for( final String name : names ) {
110                if( textualMetadataFields.contains( name ) ) {
111                    out.append( " " ).append( metadata.get( name ) );
112                }
113            }
114        } catch( final TikaException | SAXException e ) {
115            LOG.error( "Attachment cannot be parsed", e );
116        } catch( final ProviderException | IOException e ) {
117            LOG.error( "Attachment cannot be loaded", e );
118        }
119
120        return out.toString();
121    }
122
123}