.Net helper class for SQL Server Full Text Search

Michael Falconer — Wed, 10 Nov 2004 09:42:00 GMT

I've made use of SQL Servers ability to run a full text search in a couple of projects recently and wrote a little helper class that takes a typical search phrase and creates the tsql statement that you can then pass into a CONTAINSTABLE clause. Basically it looks either for words separated by a space, or a phrase with quotes around it, or a mixture of both.

There is one limitation in this implementation; the noise words are hard coded into a string array. It would be better to read in the contents of the noise word file that SQL Server uses, but in some shared hosting arrangements you may not have access to the file.

    1 using System;

    2 using System.Collections;

    4 namespace CamelJones.Data.Utils

    5 {

    6   /// <summary>

    7   /// Helper object for converting a search string into a

    8   /// Full Text Search query, used in a CONTAINSTABLE clause.

    9   /// </summary>

   10   public class SqlFtsHelper

   11   {

   12     #region Private Members

   14     private string _searchPhrase;

   15     private string _ftsQuery;

   16     private ArrayList _ignoredWords = new ArrayList();

   17     private string[] _noiseWords = new string[] {"about","after","all","also","an","and","another","any","are","as","at","be","because","been","before","being","between","both","but","by","came","can","come","could","did","do","each","for","from","get","got","has","had","he","have","her","here","him","himself","his","how","if","in","into","is","it","like","make","many","me","might","more","most","much","must","my","never","now","of","on","only","or","other","our","out","over","said","same","see","should","since","some","still","such","take","than","that","the","their","them","then","there","these","they","this","those","through","to","too","under","up","very","was","way","we","well","were","what","where","which","while","who","with","would","you","your"};

   19     #endregion

   21     #region Constructors

   23     /// <summary>

   24     /// Constructor for the SqlFtsHelper object

   25     /// </summary>

   26     /// <param name="query">The search string that is to be used to create the Full Text Search query</param>

   27     public SqlFtsHelper(string searchPhrase)

   28     {

   29       _searchPhrase = searchPhrase;

   30       BuildFTSQuery();

   31     }

   33     #endregion

   35     #region Public Methods

   37     /// <summary>

   38     /// The query string that was passed in to be used for the Full Text Search

   39     /// </summary>

   40     public string SearchPhrase

   41     {

   42       get { return _searchPhrase; }

   43     }

   45     /// <summary>

   46     /// The T-SQL statement to be used as the contains_search_condition parameter

   47     /// of a CONTAINSTABLE query

   48     /// </summary>

   49     public string FTSQuery

   50     {

   51       get { return _ftsQuery; }

   52     }

   54     /// <summary>

   55     /// An array of any noise words that were included in the search string

   56     /// </summary>

   57     public object[] IgnoredWords

   58     {

   59       get { return _ignoredWords.ToArray(); }

   60     }

   62     #endregion

   64     #region Private Methods

   66     /// <summary>

   67     /// Builds the Full Text Search query

   68     /// </summary>

   69     private void BuildFTSQuery()

   70     {

   71       _ftsQuery = "";

   72       string cleanQuery = _searchPhrase;

   74       // Strip out comments, wildcards

   75       cleanQuery = cleanQuery.Replace("--","");

   76       cleanQuery = cleanQuery.Replace("*","");

   77       cleanQuery = cleanQuery.Replace("%","");

   79       // Rather than simply split the string, walk

   80       // through it looking for phrases or individual

   81       // words

   82       cleanQuery = cleanQuery.Trim();

   83       while(cleanQuery.Length>0)

   84       {

   85         // If this is the only word

   86         if(cleanQuery.IndexOf('"')==-1 && cleanQuery.IndexOf(' ')==-1)

   87         {

   88           if(!CheckNoiseWord(cleanQuery))

   89           {

   90             FTSAddWord(cleanQuery);

   91           }

   92           break;

   93         }

   95         // Get the next word/phrase

   96         if(!cleanQuery.StartsWith("\""))

   97         {

   98           // Get the word from the search string

   99           int end = cleanQuery.IndexOf(" ");

  100           if(end<=0){end = cleanQuery.Length;}

  101           string theWord = cleanQuery.Substring(0,end);

  103           // Check it's not a noise word

  104           if(!CheckNoiseWord(theWord))

  105           {

  106             FTSAddWord(theWord);

  107           }

  109           // Remove the word from the search string

  110           cleanQuery = cleanQuery.Remove(0,end);

  111           cleanQuery = cleanQuery.Trim();

  112           continue;

  113         }

  114         else

  115         {

  116           // A phrase

  117           int end = cleanQuery.IndexOf('"',1) + 1;

  118           if(end<=1){end = cleanQuery.Length;}

  119           FTSAddPhrase(cleanQuery.Substring(0,end));

  120           cleanQuery = cleanQuery.Remove(0,end);

  121           cleanQuery = cleanQuery.Trim();

  122           continue;

  123         }

  124       }

  125     }

  127     /// <summary>

  128     /// Checks a string for noise words and, if any exist,

  129     /// removes them from the string and adds them to the

  130     /// array of ignored words.

  131     /// </summary>

  132     /// <param name="word">the word to check</param>

  133     /// <returns>true if the word is a noise word, otherwise false</returns>

  134     private bool CheckNoiseWord(string word)

  135     {

  136       if(IsNoiseWord(word))

  137       {

  138         _ignoredWords.Add(word);

  139         return true;

  140       }

  141       return false;

  142     }

  144     /// <summary>

  145     /// Add an individual word to the query

  146     /// </summary>

  147     /// <param name="word">the word to be added</param>

  148     private void FTSAddWord(string word)

  149     {

  150       if(_ftsQuery.Length>0)

  151       {

  152         _ftsQuery = _ftsQuery + " AND ";

  153       }

  154       _ftsQuery = _ftsQuery + "FORMSOF(INFLECTIONAL, " + word + ")";

  155     }

  157     /// <summary>

  158     /// Add a phrase to the FTS query

  159     /// </summary>

  160     /// <param name="phrase">the phrase to be added</param>

  161     private void FTSAddPhrase(string phrase)

  162     {

  163       if(_ftsQuery.Length>0)

  164       {

  165         _ftsQuery = _ftsQuery + " AND ";

  166       }

  167       _ftsQuery = _ftsQuery + phrase;

  168     }

  170     /// <summary>

  171     /// Checks if the word is a noise word

  172     /// </summary>

  173     /// <param name="word">the word to check</param>

  174     /// <returns>true if it is a noise word, otherwise false</returns>

  175     private bool IsNoiseWord(string word)

  176     {

  177       // Exclude individual characters

  178       if(word.Length>1)

  179       {

  180         // Iterate through the string array of noise words,

  181         // and check the word parameter does not match

  182         foreach(string item in _noiseWords)

  183         {

  184           if(word==item)

  185           {

  186             return true;

  187           }

  188         }

  190         return false;

  191       }

  193       return true;

  194     }

  196     #endregion

  197   }

  198 }

P.S. The code snippet above was formatted using the CopySourceAsHtml VS addin by Colin Coller.

New site - journalonline.co.uk

Michael Falconer — Mon, 25 Oct 2004 12:07:00 GMT

Last week a site I've been working on for a couple of months went live - www.journalonline.co.uk. Developed for the Law Society of Scotland and working with Connect Communications, the site is an online resource of information that appears in the printed magazine 'The Journal'.

The site is entirely developed in ASP.NET (using c#) and has a SQL Server 2000 backend. I made use of Paul Wilson's master pages for maintaining the site templates, and also used the Data Provider and Configuration Management application blocks. Note that if you plan to use the Configuration Management application blosk, I strongly recommend you check out the forum on the gotdotnet site where the component is hosted - there are a number of known issues with it.

The search facility provided on the site (to search the library of articles) makes use of a Full Text Index. Having used Index Server a number of times in the past to provide search facilities, I understood the ease of integration this approach would offer, and the in built facility for ranking results. It was also a low cost solution, as no third party software was required. The main downside can be the performance of the index population, but in this case that wasn't an issue as the frequency of updates was low.

The backend of the system also uses FreeTextBox (the same component used to manage this blog!).

Feel free to have a look and let me know what you think!

Exception handling in .net applications

Michael Falconer — Sat, 28 Aug 2004 19:20:00 GMT

One of the features of the .net CLR that I love the best is structured exception handling. Having worked largely on ASP / VBScript / VB applications prior to moving to C# / .net, the ability to catch and handle every type of error that can occur has proved invaluable.

Most of the .net applications I've worked on so far have been relatively simple, with very few interface boundaries and therefore I've not had to worry too much about catching exceptions until they bubble to the UI. The current project I'm working on, however, involves properly stuctured DAL / BLL and UI layers, plus I've been implementing provider patterns for a variety of components. One of my sources of information / inspiration has been the source code for .Text, where I noticed a number of classes the used a try...finally block without any catch. At first this looked a bit odd, until I came across this article on Ian Griffith's weblog.

So, if his closing remark (If it's news to you that your finally blocks should be outnumbering your catch blocks by a significant margin, then you're probably doing something wrong) applies to you, check it out.

C#

.Net helper class for SQL Server Full Text Search

New site - journalonline.co.uk

Exception handling in .net applications

Generics - List<Employee> or EmployeeList?