export GOROOT=$HOME/go
export GOOS=darwin
export GOARCH=amd64
export GOBIN=$HOME/gobin
export PATH=$PATH:$GOBIN

. ~/.bash_profile

sudo easy_install mercurial

hg clone -r release https://go.googlecode.com/hg/ $GOROOT

cd $GOROOT/src
./all.bash

package main

import fmt "fmt"

func main() {
   fmt.Printf("hello world!\n");
}

6g hello.go
6l hello.6

./6.out

hello world!

• It can operate on raw files located in HDFS, like logs from you application, like csv files from your database(s).  So this can reduce your load time, because you don't have to actually load it into a database before you can use it.
• It can operate on compressed files.  I started using this feature last week because I am getting a 4 to 1 compression ratio with no different in performance (I am using sequence files with block compression).
• In your SQL statements you can actually use the Hadoop streaming api to build your own mapper and reducers, and they don't even have to be written in Java!
• You can also create your own user defined functions, so when you have to do something crazy with the data, you can!

    /**
     * Finds the previous set bit in the bitset, start looking at the index position.
     * @param index the index to start looking.
     * @return the previous position in the bitset that is set.
     */
    public long prevSetBit(long index) {
        //get index of the word to start with
        int i = (int) (index >>> 6); //div by 64
        if (i >= wlen) {
            //if the index requested is greater than the actual
            //max word, just start with the last word
            i = wlen - 1;
        }
        //loop backwards over the bits[] until a non-zero word is found
        while (i >= 0) {
            long word = bits[i];
            if (word != 0) {
                //find the position of the most significant bit
                int pomsb = getPositionOfMostSigBit(word);
                //multiply by 64
                long l = ((long) i) << 6;
                //add together to find the actual position
                return l + pomsb;
            }
            i--;
        }
        return -1;
    }
    
    /**
     * The word passed in, should not be 0.
     * @param word the word to find the position of the most significant bit.
     * @return the position in the word.
     */
    private int getPositionOfMostSigBit(long word) {
        if (word == 1l) {
            return 0;
        }
        int i = 0;
        while (word != 1) {
            word = word >>> 1;
            i++;
        }
        return i;
    }

    public static void main(String[] args) {
        CustomOpenBitSet bitSet = new CustomOpenBitSet(Integer.MAX_VALUE);
        int maxCount = 1000000;
        fillBitSet(bitSet, maxCount);
        
        for (int i = 0; i < 10; i++) {
            double s1 = System.nanoTime();
            int slow = runSlowReverseSeeksToSetBits(bitSet, maxCount);
            double e1 = System.nanoTime();
            
            double s2 = System.nanoTime();
            int fast = runFastReverseSeeksToSetBits(bitSet, maxCount);
            double e2 = System.nanoTime();
            
            assert (slow == fast);
            
            System.out.println("The slow time took " + ((e1-s1) / ONE_BILLION) + " seconds");
            System.out.println("The fast time took " + ((e2-s2) / ONE_BILLION) + " seconds");
        }
    }

    private static int runFastReverseSeeksToSetBits(CustomOpenBitSet bitSet, int maxCount) {
        int count = 0;
        long start = Integer.MAX_VALUE;
        while (count < maxCount - 1) {
            long position = bitSet.prevSetBit(start);
            count++;
            start = position - 1;
        }
        return count;
    }

    private static int runSlowReverseSeeksToSetBits(CustomOpenBitSet bitSet, int maxCount) {
        int count = 0;
        long start = Integer.MAX_VALUE;
        while (count < maxCount - 1) {
            long position = slowSeek(bitSet,start);
            count++;
            start = position - 1;
        }
        return count;
    }

    private static long slowSeek(CustomOpenBitSet bitSet, long position) {
        while (!bitSet.get(position)) {
            position--;
            if (position < 0) {
                return -1;
            }
        }
        return position;
    }

    private static void fillBitSet(CustomOpenBitSet bitSet, int numberOfBitsToSet) {
        Random random = new Random();
        int nextInt = -1;
        for (int i = 0; i < numberOfBitsToSet; i++) {
            do {
                nextInt = random.nextInt(Integer.MAX_VALUE);
            } while (bitSet.fastGet(nextInt));
            bitSet.set(nextInt);
        }
    }

Pass [0], the slow time took 7.867899136 seconds
Pass [0], the fast time took 0.059851008 seconds
Pass [1], the slow time took 7.84461184 seconds
Pass [1], the fast time took 0.051954944 seconds
Pass [2], the slow time took 7.820731904 seconds
Pass [2], the fast time took 0.07801088 seconds
Pass [3], the slow time took 7.81953408 seconds
Pass [3], the fast time took 0.074951168 seconds
Pass [4], the slow time took 7.798106112 seconds
Pass [4], the fast time took 0.07792384 seconds
Pass [5], the slow time took 7.742498048 seconds
Pass [5], the fast time took 0.067644928 seconds
Pass [6], the slow time took 7.664754944 seconds
Pass [6], the fast time took 0.07476992 seconds
Pass [7], the slow time took 7.841472768 seconds
Pass [7], the fast time took 0.073187072 seconds
Pass [8], the slow time took 7.927188992 seconds
Pass [8], the fast time took 0.07574912 seconds
Pass [9], the slow time took 7.85273088 seconds
Pass [9], the fast time took 0.076514048 seconds

public class TestIndexerSearcher {

   public static void main(String[] args) throws Exception {
      Directory directory = new RAMDirectory();
      IndexWriter indexWriter = new IndexWriter(directory, new StandardAnalyzer());
      indexWriter.addDocument(buildDocument("DATA:sametoken","ACL_FIELD:access"));
      indexWriter.addDocument(buildDocument("DATA:sametoken","ACL_FIELD:noaccess"));
      indexWriter.optimize();
      indexWriter.close();

      IndexSearcher indexSearcher = new IndexSearcher(directory);

      QueryParser parser = new QueryParser("DATA", new StandardAnalyzer());
      Query query = parser.parse("sametoken");
		
      //This is all you have to add to your existing code.
      Filter aclFilter = applyAccessControl(new TermQuery(
         new Term("ACL_FIELD","access")));

      Hits hits = indexSearcher.search(query, aclFilter);
      System.out.println("Hits[" + hits.length() + "]");
      for (int i = 0; i < hits.length(); i++) {
         Document doc = hits.doc(i);
         System.out.println("DATA [" + doc.get("DATA") + 
            "] ACL_FIELD [" + doc.get("ACL_FIELD") + "]");
      }
      indexSearcher.close();	
   }

   private static Filter applyAccessControl(Query aclQuery) {
      return new CachedQueryFilter(aclQuery.toString(), 
         new QueryWrapperFilter(aclQuery));
   }

   private static Document buildDocument(String... fieldInfo) {
      Document document = new Document();
      for (int i = 0; i < fieldInfo.length; i++) {
         String[] split = fieldInfo[i].split(":");
         String fieldName = split[0];
         String fieldValue = split[1];
         document.add(new Field(fieldName,fieldValue,
            Field.Store.YES,Field.Index.TOKENIZED));
      }
      return document;
   }	
}

public class CachedQueryFilter extends Filter {
   private static final long serialVersionUID = 6797293376134753695L;
   private Filter filter;
   private String key;
   private static transient Map<String, BitSetCache> filterCache = 
      new ConcurrentHashMap<String, BitSetCache>();

   public CachedQueryFilter(String key, Filter filter) {
      this.filter = filter;
      this.key = key;
   }

   public BitSet bits(IndexReader reader) throws IOException {
      BitSetCache cachedBitSet = (BitSetCache) filterCache.get(key);
      if (cachedBitSet != null) {
         BitSet bitSet = cachedBitSet.bitSet.get();
         if (bitSet != null && cachedBitSet.indexReaderVersion == reader.getVersion()) {
            return bitSet;
         }
      }
      BitSet bits = filter.bits(reader);
      BitSetCache bitSetCache = new BitSetCache();
      bitSetCache.indexReaderVersion = reader.getVersion();
      bitSetCache.bitSet = new SoftReference<BitSet>(bits);
      filterCache.put(key, bitSetCache);
      return bits;
   }
	
   private class BitSetCache {
      long indexReaderVersion;
      SoftReference<BitSet> bitSet;
   }
}

Currently we index 38 fields across 1.5 billion documents, and we have implemented a fair similarity object (see Lucene Scoring Documentation) for normalized scoring. We have no use for index time field boosting or for any type of Norms (see Lucene Scoring Documentation). However Lucene reads all of the Norms into memory for fast scoring.

So let's do the math:

1 byte per Norm value * 38 fields * 1,500,000,000 = 57,000,000,000

That's +/- 57 Gigs of heap space!

That's quiet a bit of memory usage for something that we don't even use. I have since patched Lucene, so that the indexed Norms have a much better memory footprint, something around 1.5 Gigs. Not great, but livable.

I havenÕt posted my patch, because itÕs all or nothing, I havenÕt implemented a way to turn it on and off. But if there is anyone else out there, with an application that is running out of memory as users use more and more indexed fields in Lucene, take a look at the SegmentReader class. There's a byte array in there that you should take a look at. Happy hunting!

Requirements

• Fuzzy searching is a must (Soundex, Levenshtein, etc.)

• Has to be fast, a must with any searching solution

• Has to provide access control

• Full data load indexing needs to be completed in a reasonable amount of time

• Scoring needs to be a custom implementation

• Needs to run on a predetermined environment, meaning that new hardware purchases are not going to happen any time soon

• And last but not least is ability do all these things on a dataset that exceeds a billion records

So we have had a lot of constraints to deal with, the hardest one by far is the last one.

The Data

• 1 billion plus records

• Over 30 million unique terms

Indexing and Searching Server Specs

• 20 CPUs

• 32 Gig of ram

• Dedicated SAN storage

First Searching Experiences

After getting the index built in multiple partitions, I fired up a simple Lucene console to do some simple searches with a Lucene multi searcher. Ran out of memory with 2 Gig heap, tried the maximum heap size for the 32 bit JVM we were using, 3.3 Gig, and that ran out of memory as well. So, initial tries to just run one search were unsuccessful.

Then we installed a 64-bit JVM and tried an 8 Gig heap, and it worked! I could run searches and after the first couple of warm up searches it was getting 20 - 80 ms responses on single term searches. Great, but then we tried a Fuzzy search, which uses a Levenshtein algorithm to calculate matches, 2 minutes 45 seconds, this was unacceptable.

Next we wrote our own Levenshtein Lucene query and got the 2 minutes plus search down to about one second. We found that the built in Lucene Fuzzy query was taking 85-95% of the time to find the terms to search. Then after those terms were found the actual search with those expanded terms only took a second to two depending on how many terms were found. So we replaced the built in Fuzzy query with a custom one that gets near instantaneous results on Levenshtein fuzzy matches. Problem solved.

Indexing Time

After our initial proof of concept was complete, we needed to improve the indexing time down to something more reasonable. The index creation from scratch was taking 36 - 48 hours to build with 20 CPUs running at 100% utilization. Which means that the machine was indexing about 9,000 records a second. Not bad for Lucene 2.2, but not that great.

First we stopped merging the indexes after we created them, that by itself was taking about 12 hours. At this point we also started searching these multiple indexes in parallel, and we are seeing modest increases in query performance.

Second, we upgraded to Lucene 2.3, this provided a huge increase in indexing speed. Our index creation time went from 36 - 48 hours (depending on if we merged indexes or not) down to 3-4 hours. The indexing process is now indexing around 125,000+ records a second. Huge improvement, if you haven't upgraded to 2.3, you should!

Current Development

We are in the process of adding access control to Lucene as well as adding new custom queries and scoring. So far Lucene has performed better than any of the competition that it has come up against, and with it's price point it seems to have won acceptance on our project.

In upcoming parts I will go into more details about the technical solutions that we have developed to solve these problems, as well others that I haven't mentioned yet.

A column oriented database (or a column store database) is a database that stores it's information in a column oriented manor, instead of a row oriented manor.

Take for example this simple table of information.

It is very natural for a row oriented database to store the following information as sequential appending rows on disk. This makes them very good for inserting and updating data (depending on implementation).

1,John,Doe,M,1/1/1980~2,Jane,Doe,F,2/2/1981~3,John,Smith,M,3/3/1979

A column oriented database might store the data like this:

1,2,3~John,Jane,John~Doe,Doe,Smith~M,F,M~1/1/1980,2/2/1981,3/3/1979

As you can imagine this type of structure would not perform well if you were making lots of changes to the data. But if you are scanning through a column looking for values, the column oriented database groups all the values of a column together as opposed to spreading them out like the row store database would do.

Why do I care?

Database vendors would argue that indexing the columns solves all the performance problems of a row store database. And for most applications they are right, but one benefit that most column store databases provide that may not be apparent, is their ability to compress the data. Take my example:

1,2,3~John,Jane,John~Doe,Doe,Smith~M,F,M~1/1/1980,2/2/1981,3/3/1979

Because the type of each column is consistent, compression of the entire column can be a lot greater than compressing each row in a row store database (some newer RDBMs are adding compression features). Vertica, a commercial column store database boasts up to 90% reduction in storage compared with a traditional row store databases.

Query speed can also be greatly improved on a column store database versus a row store database, although that feature is more dependent on implementation then on the structure of the data.

Conclusion

For problems where you are writing and updating a lot (OLTP in database talk) stick with a row store database. But if you are reading more than you are writing (OLAP) a column store database might be something worth considering.

Here are some column store databases that I have used or heard of:

Open source:

• LucidDb - Java / JNI implementation

• MonetDb

• C-Store

Commercial:

• Vertica - Grid based column store database