Add our PIA wrapper to indri sources. This patch does several things:

 - Add pia wrapper sources to indri source tree

 - Add new tokenizer which does not treat '_' as a separator

   - The TextTokenizerPIA.l differs from TextTokenizer.l only in single character

      -[a-zA-Z0-9']+  { byte_position += tokleng; return ASCII_TOKEN; }

      +[a-zA-Z0-9_']+ { byte_position += tokleng; return ASCII_TOKEN; }

   - plus many symbol renames so that the parsers can coexist (toktext -> piatoktext etc.)

   - TextTokenizerPIA.hpp contains only symbol renamse

 - Rest are modifications to make indri build PIA wrapper

--- indri-5.4/pia_wrapper.cpp	po črc 15 14:30:41 2013

+++ indri-5.4/pia_wrapper.cpp	po črc 15 14:29:09 2013

@@ -0,0 +1,222 @@

+/*

+ * TO compile :

+ *      g++ -o libpia_wrapper.so -shared -fPIC -I../vlad-libs/sparc/usr/include/ -L../vlad-libs/sparc/usr/lib/ -lclucene-core -lnvpair pia_wrapper.cc

+ *

+ */

+

+#include <sys/stat.h>

+#include <strings.h>

+#include <stdio.h>

+#include <libnvpair.h>

+

+#include <iostream>

+#include <string>

+#include <sstream>

+#include <fstream>

+

+#include <vector>

+#include "indri/QueryEnvironment.hpp"

+#include "indri/SnippetBuilder.hpp"

+#include "indri/Repository.hpp"

+

+using namespace std;

+

+using namespace indri::api;

+

+#define MAX_RESULTS 3

+#define PIA_DATABASE "/var/db/piadb"

+#define PIA_DATABASE_STORAGE PIA_DATABASE "/collection/storage"

+

+indri::collection::Repository repository;

+

+std::string

+getFieldText(int documentID, std::string field) {

+	std::string ret_val = "";

+	indri::collection::Repository::index_state repIndexState = repository.indexes();

+	indri::index::Index *thisIndex=(*repIndexState)[0];

+	int fieldID=thisIndex->field(field);

+

+	if (fieldID < 1) {

+		return "";

+	}

+

+	const indri::index::TermList *termList=thisIndex->termList(documentID);

+

+	if (!termList) {

+		return "";

+	}

+

+	indri::utility::greedy_vector< indri::index::FieldExtent > fieldVec=termList->fields();

+	indri::utility::greedy_vector< indri::index::FieldExtent >::iterator fIter=fieldVec.begin();

+	while (fIter!=fieldVec.end()) {

+

+		if ((*fIter).id==fieldID) {

+			int beginTerm=(*fIter).begin;

+			int endTerm=(*fIter).end;

+

+	        	/*

+	 	 	 * note that the text is inclusive of the beginning

+		         * but exclusive of the ending

+		 	 */

+			for (int t=beginTerm; t < endTerm; t++) {

+				int thisTermID=termList->terms()[t];

+		       		ret_val = ret_val + thisIndex->term(thisTermID) + " ";

+			}

+		}

+

+		fIter++;

+	}

+

+	delete termList;

+	termList=NULL;

+	return ret_val;

+}

+

+/*

+ * Returns NULL on failure

+ * nvlist *

+ * search(

+ *  nvlist_t *search_params,

+ *  char **errmsg            // Similar to pia_index()

+ * );

+ */

+nvlist *

+search (nvlist_t *search_params, char **errmsg) {

+

+	char *index_path = PIA_DATABASE;

+	nvlist_t **nvl_list_result;

+	nvlist_t *nvl_return;

+	nvlist_t *nvl_result;

+	nvlist_t *results = NULL;

+

+	if (nvlist_alloc(&results, NV_UNIQUE_NAME, 0) != 0) {

+		*errmsg = strdup("nvlist_alloc failed\n");

+		return NULL;

+	}

+

+	try {

+		std::string query;

+		char *panicstack;

+		(void) nvlist_lookup_string(search_params, "stack", &panicstack);

+

+		QueryEnvironment indriEnvironment;

+		indriEnvironment.addIndex(index_path);

+

+		/* Create Indri query */

+		query = "#combine (" + std::string(panicstack) + ")";

+

+		QueryAnnotation *QAresults=indriEnvironment.runAnnotatedQuery(query.c_str(), MAX_RESULTS);

+

+		std::vector<indri::api::ScoredExtentResult> resultVector=QAresults->getResults();

+

+		int totalNumResults=resultVector.size();

+

+		/* Get Parsed document of the results */

+		std::vector<ParsedDocument*> parsedDocs=indriEnvironment.documents(resultVector);

+

+		int results_to_return = 0;

+		for ( size_t i=0; i < totalNumResults && i < MAX_RESULTS; i++ ) {

+				results_to_return++;

+		}

+

+		/* Open Repository */

+		repository.openRead(index_path);

+

+		nvl_list_result = (nvlist_t **) malloc(results_to_return * sizeof(nvlist_t *));

+

+		for ( size_t i=0; i < results_to_return; i++ ) {

+

+			std::string ret="";

+

+			int thisResultDocID=resultVector[i].document;

+

+			if (nvlist_alloc(&nvl_list_result[i], NV_UNIQUE_NAME, 0) != 0) {

+				*errmsg = strdup("nvlist_alloc failed\n");

+				return NULL;

+			}

+

+			if ((ret = getFieldText(thisResultDocID, "bug")) == "") {

+				*errmsg = strdup("Lookup of bugid failed\n");

+				return NULL;

+			} else if (nvlist_add_string(nvl_list_result[i], "pia-bugid", ret.c_str())) {

+				*errmsg = strdup("nvlist_add bugid failed\n");

+				return NULL;

+			}

+

+			if ((ret = getFieldText(thisResultDocID, "stack")) == "") {

+				*errmsg = strdup("Lookup of stack failed\n");

+				return NULL;

+			} else if (nvlist_add_string(nvl_list_result[i], "pia-stack", ret.c_str())) {

+				*errmsg = strdup("nvlist_add stack failed\n");

+				return NULL;

+			}

+

+			if ((ret = getFieldText(thisResultDocID, "signature")) == "") {

+				*errmsg = strdup("Lookup of signature failed\n");

+				return NULL;

+			} else if (nvlist_add_string(nvl_list_result[i], "pia-signature", ret.c_str())) {

+				*errmsg = strdup("nvlist_add signature failed\n");

+				return NULL;

+			}

+

+			int indri_score = 1000 + (int)resultVector[i].score*1000;

+			if (nvlist_add_int32(nvl_list_result[i], "pia-score", indri_score)) {

+				*errmsg = strdup("nvlist_add score failed\n");

+				return NULL;

+			}

+		}

+		repository.close();

+

+		nvlist_add_nvlist_array(results, "results", nvl_list_result, results_to_return);

+

+		for (int i=0; i<results_to_return; i++) {

+			nvlist_free(nvl_list_result[i]);

+		}

+

+		return results;

+

+	} catch(...){

+		nvl_list_result = (nvlist_t **) malloc(1 * sizeof(nvlist_t **));

+

+		if (nvlist_alloc(&nvl_result, NV_UNIQUE_NAME, 0) != 0) {

+			*errmsg = strdup("nvlist_alloc failed\n");

+			return NULL;

+		}

+

+		if (nvlist_add_string(nvl_result, "error", "Indri Error")) {

+			*errmsg = strdup("nvlist_add error failed\n");

+			return NULL;

+                }

+

+		nvlist_dup(nvl_result, &nvl_list_result[0], 0);

+		nvlist_free(nvl_result);

+		nvlist_add_nvlist_array(results, "results", nvl_list_result, 1);

+

+		return results;

+        }

+}

+

+extern "C" nvlist*

+pia_search (nvlist_t *search_params, char **errmsg) {

+

+	return search (search_params, errmsg);

+

+}

+

+int

+init () {

+

+	struct stat sb;

+	if (stat(PIA_DATABASE_STORAGE, &sb) != 0) {

+		return 1;

+	}

+

+	return 0;

+}

+

+extern "C" int

+pia_init () {

+

+	return init ();

+

+}

--- indri-5.4/src/TextTokenizerPIA.l	po črc 15 14:38:12 2013

+++ indri-5.4/src/TextTokenizerPIA.l	po črc 15 14:36:55 2013

@@ -0,0 +1,588 @@

+%option noyywrap

+%option never-interactive

+%option prefix="piatok"

+

+%{

+

+/*==========================================================================

+ * Copyright (c) 2004 University of Massachusetts.  All Rights Reserved.

+ *

+ * Use of the Lemur Toolkit for Language Modeling and Information Retrieval

+ * is subject to the terms of the software license set forth in the LICENSE

+ * file included with this software, and also available at

+ * http://www.lemurproject.org/license.html

+ *

+ *==========================================================================

+ */

+

+//

+// TextTokenizerPIA

+//

+// 15 September 2005 -- mwb

+//

+

+#include <string.h>

+#include <ctype.h>

+#include "indri/TextTokenizerPIA.hpp"

+#include "indri/TermExtent.hpp"

+#include "indri/TagEvent.hpp"

+#include "indri/TokenizedDocument.hpp"

+#include "indri/UnparsedDocument.hpp"

+#include "indri/UTF8Transcoder.hpp"

+#include "indri/AttributeValuePair.hpp"

+

+static long byte_position;

+

+#define ZAP           1

+#define TAG           2

+#define ASCII_TOKEN   3

+#define UTF8_TOKEN    4

+

+%}

+%start COMMENT

+%%

+

+"<!--" { BEGIN(COMMENT); byte_position += piatokleng; return ZAP; }

+<COMMENT>[^-]+ { byte_position += piatokleng; return ZAP; }

+<COMMENT>"-->" { BEGIN(INITIAL); byte_position += piatokleng; return ZAP; }

+<COMMENT>"-" { byte_position += piatokleng; return ZAP; }

+"<!"[^\>]*">" { byte_position += piatokleng; return ZAP; }

+\<[a-zA-Z/][^\>]*\>                                             { byte_position += piatokleng; return TAG; }

+[&]([a-zA-Z]+|[#]([0-9]+|[xX][a-fA-F0-9]+))[;]         { byte_position += piatokleng; return ZAP; /* symbols */ }

+[A-Z0-9]"."([A-Z0-9]".")*                                        { byte_position += piatokleng; return ASCII_TOKEN; }

+[a-zA-Z0-9_']+                                        { byte_position += piatokleng; return ASCII_TOKEN; }

+"-"[0-9]+("."[0-9]+)?                                  { byte_position += piatokleng; return ASCII_TOKEN; }

+[a-zA-Z0-9\x80-\xFD]+                               { byte_position += piatokleng; return UTF8_TOKEN; }

+

+[\n]                                                   { byte_position += piatokleng; return ZAP; }

+.                                                      { byte_position += piatokleng; return ZAP; }

+

+%%

+

+indri::parse::TokenizedDocument* indri::parse::TextTokenizerPIA::tokenize( indri::parse::UnparsedDocument* document ) {

+

+  _termBuffer.clear();

+  if ( _tokenize_entire_words)

+    _termBuffer.grow( document->textLength * 4);

+  else

+    _termBuffer.grow( document->textLength * 8 ); // extra null per char.

+

+  _document.terms.clear();

+  _document.tags.clear();

+  _document.positions.clear();

+

+  _document.metadata = document->metadata;

+  _document.text = document->text;

+  _document.textLength = document->textLength;

+  _document.content = document->content;

+  _document.contentLength = document->contentLength;

+

+  // byte offset

+  byte_position = document->content - document->text;

+

+  piatok_scan_bytes( document->content, document->contentLength );

+

+  // Main Tokenizer loop

+

+  int type;

+

+  while ( type = piatoklex() ) {

+

+    switch ( type ) {

+

+    case ASCII_TOKEN: processASCIIToken(); break;

+

+    case UTF8_TOKEN: processUTF8Token(); break;

+

+    case TAG: if ( _tokenize_markup ) processTag(); break;

+

+    default:

+    case ZAP:

+      break;

+

+    }

+

+  }

+

+  piatok_delete_buffer( YY_CURRENT_BUFFER );

+

+  return &_document;

+}

+

+// Member functions for processing tokenization events as dispatched

+// from the main tokenizer loop

+

+void indri::parse::TextTokenizerPIA::processTag() {

+

+  // Here, we parse the tag in a fashion that is relatively robust to

+  // malformed markup.  toktext matches this pattern: <[^>]+>

+

+  if ( piatoktext[1] == '?' || piatoktext[1] == '!' ) {

+

+    // XML declaration like <? ... ?> and <!DOCTYPE ... >

+    return; // ignore

+

+  } else if ( piatoktext[1] == '/' ) { // close tag, eg. </FOO>

+

+    // Downcase the tag name.

+

+    int len = 0;

+

+    for ( char *c = piatoktext + 2;

+#ifndef WIN32

+          isalnum( *c ) || *c == '-' || *c == '_' || *c == ':' ; c++ ) {

+#else

+          ((*c >= 0) && isalnum( *c )) || *c == '-' || *c == '_' || *c == ':' ; c++ ) {

+#endif

+

+      *c = tolower( *c );

+      if ( *c == ':' ) *c = '_'; /* replace colon (from namespaces) */

+      len++;

+    }

+

+    TagEvent te;

+

+    te.open_tag = false;

+

+    // We need to write len characters, plus a NULL

+    char* write_loc = _termBuffer.write( len + 1 );

+    strncpy( write_loc, piatoktext + 2, len );

+    write_loc[len] = '\0';

+    te.name = write_loc;

+

+    // token position of tag event w/r/t token string

+    te.pos = _document.terms.size();

+

+    te.begin = byte_position - piatokleng;

+    te.end = byte_position;

+

+    _document.tags.push_back( te );

+

+#ifndef WIN32

+    } else if ( isalpha( piatoktext[1] ) ) {

+#else

+    } else if ( (piatoktext[1]  >= 0) && (isalpha( piatoktext[1] ) )) {

+#endif

+

+    // Try to extract the tag name:

+

+    char* c = piatoktext + 1;

+    int i = 0;

+    int offset = 1; // current offset w/r/t byte_position - piatokleng

+    // it starts at one because it is incremented when c is, and c starts at one.

+    char* write_loc;

+

+#ifndef WIN32

+    while ( isalnum( c[i] ) || c[i] == '-' || c[i] == '_' || c[i] == ':' ) i++;

+#else

+    while ( ( (c[i] >= 0) && isalnum( c[i] )) || c[i] == '-' || c[i] == '_' || c[i] == ':' ) i++;

+#endif

+    if ( c[i] == '>' ) {

+

+      // open tag with no attributes, eg. <title>

+

+      // Ensure tag name is downcased

+      for ( int j = 0; j < i; j++ ) {

+        c[j] = tolower( c[j] );

+        if ( c[j] == ':' ) c[j] = '_'; /* replace colon (from namespaces) */

+      }

+

+      TagEvent te;

+

+      te.open_tag = true;

+

+      // need to write i characters, plus a NULL

+      char* write_loc = _termBuffer.write( i + 1 );

+      strncpy( write_loc, c, i );

+      write_loc[i] = '\0';

+      te.name = write_loc;

+

+      te.pos = _document.terms.size();

+

+      te.begin = byte_position - piatokleng;

+      te.end = byte_position;

+

+      _document.tags.push_back( te );

+

+#ifndef WIN32

+    } else if ( isspace( c[i] ) ) {

+#else

+    } else if ( (c[i]  >= 0) && (isspace( c[i] ) )) {

+#endif

+

+      // open tag with attributes, eg. <A HREF="www.foo.com/bar">

+

+      TagEvent te;

+

+      te.open_tag = true;

+

+      // Ensure tag name is downcased

+      for ( int j = 0; j < i; j++ ) {

+        c[j] = tolower( c[j] );

+        if ( c[j] == ':' ) c[j] = '_'; /* replace colon (from namespaces) */

+      }

+

+      // need to write i characters, plus a NULL

+      char* write_loc = _termBuffer.write( i + 1 );

+      strncpy( write_loc, c, i );

+      write_loc[i] = '\0';

+      te.name = write_loc;

+      c += i;

+      offset += i;

+

+#ifndef WIN32

+    while ( isspace( *c ) ) { c++; offset++; }

+#else

+    while (((*c) >=0) &&  isspace( *c )) { c++; offset++; }

+#endif

+

+      te.pos = _document.terms.size();

+

+      te.begin = byte_position - piatokleng;

+      te.end = byte_position;

+

+      // Now search for attributes:

+

+      while ( *c != '>' && *c != '\0' ) {

+

+        AttributeValuePair avp;

+

+        // Try to extract attribute name:

+

+        i = 0;

+#ifndef WIN32

+        while ( isalnum( c[i] ) || c[i] == '-' || c[i] == '_' ) i++;

+#else

+        while ( (c[i] >= 0) && isalnum( c[i] ) || c[i] == '-' || c[i] == '_') i++;

+#endif

+

+        if ( i == 0 ) break;

+

+        // Ensure attribute name is downcased

+        for ( int j = 0; j < i; j++ )

+          c[j] = tolower( c[j] );

+

+        // need to write i characters, plus a NULL

+        write_loc = _termBuffer.write( i + 1 );

+        strncpy( write_loc, c, i );

+        write_loc[i] = '\0';

+        avp.attribute = write_loc;

+        c += i;

+        offset += i;

+

+        // attributes can be foo\s*=\s*"bar[">] or foo\s*=\s*bar

+

+		// ignore any spaces

+#ifndef WIN32

+    while ( isspace( *c ) ) { c++; offset++; }

+#else

+    while (((*c) >=0) &&  isspace( *c )) { c++; offset++; }

+#endif

+

+        if ( *c == '=' ) {

+

+          c++; // get past the '=' sign.

+          offset++;

+

+#ifndef WIN32

+    while ( isspace( *c ) ) { c++; offset++; }

+#else

+    while (((*c) >=0) &&  isspace( *c )) { c++; offset++; }