SQL Server

.Net helper class for SQL Server Full Text Search

Michael Falconer — Wed, 10 Nov 2004 09:42:00 GMT

I've made use of SQL Servers ability to run a full text search in a couple of projects recently and wrote a little helper class that takes a typical search phrase and creates the tsql statement that you can then pass into a CONTAINSTABLE clause. Basically it looks either for words separated by a space, or a phrase with quotes around it, or a mixture of both.

There is one limitation in this implementation; the noise words are hard coded into a string array. It would be better to read in the contents of the noise word file that SQL Server uses, but in some shared hosting arrangements you may not have access to the file.

    1 using System;

    2 using System.Collections;

    4 namespace CamelJones.Data.Utils

    5 {

    6   /// <summary>

    7   /// Helper object for converting a search string into a

    8   /// Full Text Search query, used in a CONTAINSTABLE clause.

    9   /// </summary>

   10   public class SqlFtsHelper

   11   {

   12     #region Private Members

   14     private string _searchPhrase;

   15     private string _ftsQuery;

   16     private ArrayList _ignoredWords = new ArrayList();

   17     private string[] _noiseWords = new string[] {"about","after","all","also","an","and","another","any","are","as","at","be","because","been","before","being","between","both","but","by","came","can","come","could","did","do","each","for","from","get","got","has","had","he","have","her","here","him","himself","his","how","if","in","into","is","it","like","make","many","me","might","more","most","much","must","my","never","now","of","on","only","or","other","our","out","over","said","same","see","should","since","some","still","such","take","than","that","the","their","them","then","there","these","they","this","those","through","to","too","under","up","very","was","way","we","well","were","what","where","which","while","who","with","would","you","your"};

   19     #endregion

   21     #region Constructors

   23     /// <summary>

   24     /// Constructor for the SqlFtsHelper object

   25     /// </summary>

   26     /// <param name="query">The search string that is to be used to create the Full Text Search query</param>

   27     public SqlFtsHelper(string searchPhrase)

   28     {

   29       _searchPhrase = searchPhrase;

   30       BuildFTSQuery();

   31     }

   33     #endregion

   35     #region Public Methods

   37     /// <summary>

   38     /// The query string that was passed in to be used for the Full Text Search

   39     /// </summary>

   40     public string SearchPhrase

   41     {

   42       get { return _searchPhrase; }

   43     }

   45     /// <summary>

   46     /// The T-SQL statement to be used as the contains_search_condition parameter

   47     /// of a CONTAINSTABLE query

   48     /// </summary>

   49     public string FTSQuery

   50     {

   51       get { return _ftsQuery; }

   52     }

   54     /// <summary>

   55     /// An array of any noise words that were included in the search string

   56     /// </summary>

   57     public object[] IgnoredWords

   58     {

   59       get { return _ignoredWords.ToArray(); }

   60     }

   62     #endregion

   64     #region Private Methods

   66     /// <summary>

   67     /// Builds the Full Text Search query

   68     /// </summary>

   69     private void BuildFTSQuery()

   70     {

   71       _ftsQuery = "";

   72       string cleanQuery = _searchPhrase;

   74       // Strip out comments, wildcards

   75       cleanQuery = cleanQuery.Replace("--","");

   76       cleanQuery = cleanQuery.Replace("*","");

   77       cleanQuery = cleanQuery.Replace("%","");

   79       // Rather than simply split the string, walk

   80       // through it looking for phrases or individual

   81       // words

   82       cleanQuery = cleanQuery.Trim();

   83       while(cleanQuery.Length>0)

   84       {

   85         // If this is the only word

   86         if(cleanQuery.IndexOf('"')==-1 && cleanQuery.IndexOf(' ')==-1)

   87         {

   88           if(!CheckNoiseWord(cleanQuery))

   89           {

   90             FTSAddWord(cleanQuery);

   91           }

   92           break;

   93         }

   95         // Get the next word/phrase

   96         if(!cleanQuery.StartsWith("\""))

   97         {

   98           // Get the word from the search string

   99           int end = cleanQuery.IndexOf(" ");

  100           if(end<=0){end = cleanQuery.Length;}

  101           string theWord = cleanQuery.Substring(0,end);

  103           // Check it's not a noise word

  104           if(!CheckNoiseWord(theWord))

  105           {

  106             FTSAddWord(theWord);

  107           }

  109           // Remove the word from the search string

  110           cleanQuery = cleanQuery.Remove(0,end);

  111           cleanQuery = cleanQuery.Trim();

  112           continue;

  113         }

  114         else

  115         {

  116           // A phrase

  117           int end = cleanQuery.IndexOf('"',1) + 1;

  118           if(end<=1){end = cleanQuery.Length;}

  119           FTSAddPhrase(cleanQuery.Substring(0,end));

  120           cleanQuery = cleanQuery.Remove(0,end);

  121           cleanQuery = cleanQuery.Trim();

  122           continue;

  123         }

  124       }

  125     }

  127     /// <summary>

  128     /// Checks a string for noise words and, if any exist,

  129     /// removes them from the string and adds them to the

  130     /// array of ignored words.

  131     /// </summary>

  132     /// <param name="word">the word to check</param>

  133     /// <returns>true if the word is a noise word, otherwise false</returns>

  134     private bool CheckNoiseWord(string word)

  135     {

  136       if(IsNoiseWord(word))

  137       {

  138         _ignoredWords.Add(word);

  139         return true;

  140       }

  141       return false;

  142     }

  144     /// <summary>

  145     /// Add an individual word to the query

  146     /// </summary>

  147     /// <param name="word">the word to be added</param>

  148     private void FTSAddWord(string word)

  149     {

  150       if(_ftsQuery.Length>0)

  151       {

  152         _ftsQuery = _ftsQuery + " AND ";

  153       }

  154       _ftsQuery = _ftsQuery + "FORMSOF(INFLECTIONAL, " + word + ")";

  155     }

  157     /// <summary>

  158     /// Add a phrase to the FTS query

  159     /// </summary>

  160     /// <param name="phrase">the phrase to be added</param>

  161     private void FTSAddPhrase(string phrase)

  162     {

  163       if(_ftsQuery.Length>0)

  164       {

  165         _ftsQuery = _ftsQuery + " AND ";

  166       }

  167       _ftsQuery = _ftsQuery + phrase;

  168     }

  170     /// <summary>

  171     /// Checks if the word is a noise word

  172     /// </summary>

  173     /// <param name="word">the word to check</param>

  174     /// <returns>true if it is a noise word, otherwise false</returns>

  175     private bool IsNoiseWord(string word)

  176     {

  177       // Exclude individual characters

  178       if(word.Length>1)

  179       {

  180         // Iterate through the string array of noise words,

  181         // and check the word parameter does not match

  182         foreach(string item in _noiseWords)

  183         {

  184           if(word==item)

  185           {

  186             return true;

  187           }

  188         }

  190         return false;

  191       }

  193       return true;

  194     }

  196     #endregion

  197   }

  198 }

P.S. The code snippet above was formatted using the CopySourceAsHtml VS addin by Colin Coller.

New site - journalonline.co.uk

Michael Falconer — Mon, 25 Oct 2004 12:07:00 GMT

Last week a site I've been working on for a couple of months went live - www.journalonline.co.uk. Developed for the Law Society of Scotland and working with Connect Communications, the site is an online resource of information that appears in the printed magazine 'The Journal'.

The site is entirely developed in ASP.NET (using c#) and has a SQL Server 2000 backend. I made use of Paul Wilson's master pages for maintaining the site templates, and also used the Data Provider and Configuration Management application blocks. Note that if you plan to use the Configuration Management application blosk, I strongly recommend you check out the forum on the gotdotnet site where the component is hosted - there are a number of known issues with it.

The search facility provided on the site (to search the library of articles) makes use of a Full Text Index. Having used Index Server a number of times in the past to provide search facilities, I understood the ease of integration this approach would offer, and the in built facility for ranking results. It was also a low cost solution, as no third party software was required. The main downside can be the performance of the index population, but in this case that wasn't an issue as the frequency of updates was low.

The backend of the system also uses FreeTextBox (the same component used to manage this blog!).

Feel free to have a look and let me know what you think!

All Tied Up

Michael D. Falconer — Mon, 07 Jun 2004 22:37:00 GMT

While studying for my SQL Server exam recently I came across an interesting feature of the T-SQL SELECT statement I'd never noticed before - the WITH TIES argument. Used in conjunction with a TOP n or TOP n PERCENT argument, and the ORDER BY clause, you can return not just the n records you'd expect, but all additional records with the same value in the ORDER BY column as the n^th record.

Lets say, for example, that you have an online competition and you want to pick the top 5 entrants who achieved the highest scores. Your data might include the following entries:

Score    Name
285       Bob
250       Bill
240       Mick
240       Sue
235       James
235       Jill
235       Dean
230       Michelle

In the past I would of just written:

SELECT TOP 5 * FROM tablename ORDER BY score DESC

However, how would this decide between James, Jill and Dean who are in join 5th place? It would be much better to include those entries who are in joint 5th position (unless, of course, you also decide to order the results by the date entered so the first person to get the score wins - but let's say we don't!). Instead of the above, you can write the query as:

SELECT TOP 5 WITH TIES * FROM tablename ORDER BY score DESC

This will return not just the top 5 records, but also additional records with the same value as the same score as the 5^th record. In this example, it will return all records from Bob to Dean - 7 records in total.

Rob Howard : Farewell and Teched slides

Michael D. Falconer — Tue, 01 Jun 2004 13:02:00 GMT

Rob Howard has announced that he's leaving Microsoft, but has thankfully posted up slides from some TechEd US presentations. I've just had a flick through the presentation titled 'Running www.asp.net', and it looks very interesting. There's certainly a nice list of recommendations to follow.

70-229 : SQL Server 2000 Exam

Michael D. Falconer — Tue, 01 Jun 2004 12:00:00 GMT

Well, I passed my SQL Server exam (70-229) yesterday, and I'm glad it's over! I managed to score 853 (I assume out of 1000), so that's not too bad. Congratulations also to Jim, who managed to pass his VB.NET Web Applications at the same time.

SQL Server Exam

Michael D. Falconer — Thu, 27 May 2004 11:09:00 GMT

I finally decided to sit my first Microsoft exam, and am booked in to sit Exam 70-229 : Designing and implementing databases with Microsoft SQL Server 2000 Enterprise Edition on Monday. Not looking forward to it, mainly because I've had very little time to study. Guess I'll have to see how it goes - if I fail, at least I'll know what to expect for the other exams!

I've never been particularly bothered by the exams before, but now that I'm doing freelance/contract work, it makes sense to gain MCSD certification so that potential clients can see that I really can do most of what I say I can! Maybe I should of gone for the Windows or Web applciations exams first, but I've worked with SQL for longer than .net, so it kind of makes sense to try it first.

If it all goes quiet from me on Monday, I'll be hangnig my head in shame, and hiding in a small dark corner of some room...