Description

PowerShell script that uses a input configuration file to set default query and connection parameters to execute search against query server host in a FS4SP setup.

Returned relevance log from search result is parsed and presented in a human readable way

Script

PowerShell
Edit|Remove
#
# Script used to run on QR server host in FS4SP solutions
# Script will based on input file (search result file with ranklog) or input FQL 
# query parse the ranklog and present details in a more readable format.
# Understanding the output form the ranklog is important in tuning and testing of search relevancy.
#
# Author:  Jan Inge Bergseth (janib@microsoft.com)
#

function Shift-Right ($x =1, $y)
{
    $result = $x
    (1..$y)|%{$result = [Math]::Floor($result / 2)}
    return $result
}

#
# Convert context map (binary value, as decimal) to levels where context is found
#
function context([string]$inp)
{
    $digit = 2147483648
    $map   = ""
    
    for ($i=0; $i -lt 32; $i++)
    {
        if (($inp -band $digit) -ne 0) 
        {
            $map =  $map + (32 - $i) + ", "
        }
        $digit = Shift-Right $digit 1
    }
    $map = $map.Substring(0,$map.LastIndexOf(','))
    return $map
}


#
# *** Main ***
# 


$curDir = Get-Location
$configFile = "ranklogparser.config"

if (Test-Path $configFile)
{
    $defHits = ""
    $defSort = ""
    foreach ($line in get-content $configFile) 
    {
        if (!$line.StartsWith('#'))
        { 
            if ($line.StartsWith("inputFile"))
            { 
                $defFile = $line.Substring($line.IndexOf('=')+1)
            }
            elseif ($line.StartsWith("query"))
            {
                $defQuery = $line.Substring($line.IndexOf('=')+1)
            } 
            elseif ($line.StartsWith("hits"))
            {
                $defHits = $line.Substring($line.IndexOf('=')+1)
            } 
            elseif ($line.StartsWith("sort"))
            {
                $defSort = $line.Substring($line.IndexOf('=')+1)
            } 
            elseif ($line.StartsWith("security"))
            {
                $defSecurity = $line.Substring($line.IndexOf('=')+1)
            } 
            elseif ($line.Trim() -ne "") 
            {
                Write-Host "Unknown parameter:" $line            
            }
        }
    }
}
else
{
    Write-Host "Did not find any '" $configFile "' file with default parameters in current dir ("$curDir")"
}

# Read input param
$type = Read-Host "Run with new query(q) or existing ranklog file(f) as input (q/f)"
if ($type -eq "")
{
    Write-Host "No type provided, using input file as default..."
    $type = "f"
}


$sinput = @()
if ($type -eq "f")
{
    $inputFile = Read-Host "Input ranklog file (default :" $defFile ")"
    if ($inputFile -eq "")
    {
        if ($defFile -eq "")
        {
            Write-Host "ERROR: Missing input ranglog file to parse"
            Write-Host "ERROR: Create it by running search on localhost, ex: "
            Write-Host "       http://localhost:13280/cgi-bin/search?offset=0&hits=10&query=and(public,sector)&ranklog"
            Exit
        }
        $inputFile = $defFile
    }
    foreach ($line in get-content $inputFile) 
    {
        $sinput = $sinput + $line
    }
}
else
{
    $query = Read-Host "FQL query (default:" $defQuery ")"
    if ($query -eq "")
    {
        if ($defQuery -eq "")
        {
            Write-Host "ERROR: Missing FQL input: query"
            Write-Host "ERROR: Example query: and(public,sector)"
            Exit
        }
        $query = $defQuery
    }
    $query = $query.trim()

    if ($defHits -eq "")
    {
        $defHits = 10
    }
    $hits  = Read-Host "Hits: ("$defHits ")"
    if ([int]$hits -eq 0)
    {
        $hits = $defHits
    }  
    if ($defSort -eq "")
    {
        $defSort = "default"
    }
    
    $sort = Read-Host "Sortby: ("$defSort ")"
    if ($sort -eq "")
    {
        $sort = $defSort   
    } 
    $security  = Read-Host "Security Claim token: ("$defSecurity")"
    if ($security -ne "")
    {
        $security = "&qtf_securityfql:uid=" + $security
    } 
    
    $url = "http://localhost:13280/cgi-bin/search?offset=0&hits=" +$hits+ "&query=" +$query+ "&ranklog" + "&sortby=" + $sort + $security
    Write-Host "URL:" $url

    $xHTTP = new-object -com msxml2.xmlhttp;
    $xHTTP.open("GET",$url,$false);
    $xHTTP.send();
    
    $sinput = $xHTTP.ResponseText.Split("`n")
    if ($sinput.Length -le 10)
    {
        $xHTTP.ResponseText
        Exit
    }
}



$writeQuery = 0
foreach ($line in $sinput) 
{
    
    if ($line.StartsWith("#QTF NAM Final query"))
    {
        $writeQuery = 1
    }
    elseif ($line.StartsWith("#QTF QRY ") -and ($writeQuery -eq 1))
    {
        $query = $line.Substring(9)
        $q = @()
        $q = [regex]::Split( $query, "\),\s" )
        $num = $q.length
        $i = 0
        write-Host "********************* QUERY **********************************************"
        foreach ($ql in $q)
        {
            $i += 1
            if ($i -lt $num) 
            {
                $ql = $ql.Trim() + "),"
            }
            write-Host $ql
            
        }
        write-Host "**************************************************************************"
    }

    #read ranklog
    elseif ($line.StartsWith("#ranklog"))  
    {
        $rank = $line
        
		$res = @()
        $res = [regex]::Split( $rank, "\sOP\s" )
        #$res = ([regex]' OP (.+?)( OP )|( ENDRANKLOG)').matches($rank) | foreach {$_.Groups[1].Value}        
        
        #$res
        write-Host "**************************************************************************"
        foreach ($r in $res)
        {
            if ($r.Contains("firstocc=") -and !$r.Contains("msynthcat"))
            {
                $term    = ([regex]'\:(.+?) TERMINDEX').matches($r) | foreach {$_.Groups[1].Value} 
                $term    =  $term.Replace('T','')
                
                $calc  =  ([regex]'\(\(\(\((.+?)\)\*').matches($r) | foreach {$_.Groups[1].Value}  
                $fnum     = [int]$calc.Substring(0,$calc.IndexOf("+"))
                $firstocc = $r.Substring($r.IndexOf(" firstocc=")+10)
                $firstocc = $firstocc.Substring(0, $firstocc.IndexOf(' '))

                $calc     = $calc.Substring($calc.IndexOf("+")+1)
                $nnum     = [int]$calc.Substring(0,$calc.IndexOf("+"))
                $numoccs = $r.Substring($r.IndexOf(" numoccs=")+9)
                $numoccs = $numoccs.Substring(0, $numoccs.IndexOf(' '))

                $calc     = $calc.Substring($calc.IndexOf("+")+1)
                $enum     = [int]$calc.Substring(0,$calc.IndexOf("+"))
                
                $extnumoccs = $r.Substring($r.LastIndexOf(" extnumoccs=")+12)
                $extnumoccs = $extnumoccs.Substring(0, $extnumoccs.IndexOf(' '))
                $xnum     = [int]$calc.Substring($calc.IndexOf("+")+1)
                
                $contextmap = $r.Substring($r.LastIndexOf(" contextmap=")+12)
                $contextmap = $contextmap.Substring(0, $contextmap.IndexOf(' '))
                

                
                $level = context $contextmap
                $allboost = $r.Substring($r.LastIndexOf(' ')+1)
                
                $allboost  = $allboost.Substring($allboost.IndexOf('+')+1)
                $context   = $allboost.Substring(0, $allboost.IndexOf('+'))
                $allboost  = $allboost.Substring($allboost.IndexOf('+')+1)
                $proximity = $allboost.Substring(0, $allboost.IndexOf('+'))
                $allboost  = $allboost.Substring($allboost.IndexOf('+')+1)
                $comcontx  = $allboost.Substring(0, $allboost.IndexOf('+'))
                $allboost  = $allboost.Substring($allboost.IndexOf('+')+1)
                $oppboost  = $allboost.Substring(0, $allboost.IndexOf('='))
                
                $xsum = [int]$context/($fnum + $nnum + $enum + $xnum)
                
                
                write-Host "Query term: " $term.Trim()
                write-Host "  Context score...............: " $context
                if ($fnum -ne 0)
                {
                    $fnum = [math]::round(($fnum * $xsum), 0)
                    write-Host "    First occurence position/score......: " $firstocc "/" $fnum
                }
                if ($nnum -ne 0)
                {
                    $nnum = [math]::round(($nnum * $xsum), 0)
                    write-Host "    Number of hits/score in context.....: " $numoccs "/" $nnum
                }
                if ($enum -ne 0)
                {
                    $enum = [math]::round(($enum * $xsum), 0)
                    write-Host "    Anchor & query text hits/score in...: " $extnumoccs "/" $enum
                }
                if ($xnum -ne 0)
                {
                    $xnum = [math]::round(($xnum * $xsum), 0)
                    write-Host "    Context level/score.................: " $level "/" $xnum
                }
                if ($proximity -ne 0)
                {
                    write-Host "  Proximity score.............: " $proximity
                }
                if ($comcontx -ne 0)
                {
                    write-Host "  Common-context score........: " $comcontx
                }
                if ($oppboost -ne 0)
                {
                    write-Host "  Operator score..............: " $oppboost
                }
                
            }
            elseif ($r.StartsWith("PHRASE,"))  
            {
                $ph    = ([regex]'\:(.+?) TERMINDEX').matches($r) | foreach {$_.Groups[1].Value} 
                $ph    =  $ph.Replace('T','')
                $ph    =  $ph.Replace("bt1.bidx",'')
            }         
            elseif ($r.StartsWith("XRANK,"))  
            {
                $rank = [long]$r.Substring($r.LastIndexOf('=')+1)
                if ($rank -gt 2147483648)
                {
                    $rank = $rank - 4294967296
                    $ph = " Term: "+ $ph + " (subtracted from Context boost)"
                }
                else
                {
                    $ph = "     Term: "+ $ph                
                }
                write-Host  "XRANK score...................: " $rank  $ph
            }
            elseif ($r.StartsWith("XRANKTERM,"))  
            {
   		        $ph = @()                
                $ph = ([regex]'\.bidx(.+?)T').matches($r) | foreach {$_.Groups[1].Value}  
                $rank = [long]$r.Substring($r.LastIndexOf('=') +1)
                if ($rank -gt 2147483648)
                {
                    $rank = $rank - 4294967296
                    $ph = " Term: "+ $ph + " (subtracted from Context boost)"
                }
                else
                {
                    $ph = "     Term: "+ $ph                
                }
                write-Host  "XRANK score...................: " $rank  $ph
            }
            elseif ($r.StartsWith("STATICRANK,"))  
            {
                $rank = $r.Substring($r.LastIndexOf(")=")+2)
                $rank = $rank.Substring(0,$rank.IndexOf(')'))
                
                write-Host "Static rank score.............: " $rank
            }
            
            elseif ($r.StartsWith("FRESHNESS,"))  
            {
                $rank = $r.Substring($r.LastIndexOf(")=")+2)
                $totrank = $r.Substring($r.LastIndexOf('=')+1)
                
                $rank = $rank.Substring(0,$rank.IndexOf(')'))
                $totrank = $totrank.Substring(0,$totrank.LastIndexOf(' '))
                
                write-Host "Freshness score...............: " $rank
                write-Host "Total Rank scorescore.........: " $totrank
            }
            
        }
    }
    elseif ($line.StartsWith("#### "))  
    {
        $hit = $line.Substring(5)
    }
    elseif ($line.StartsWith("#url "))  
    {
        write-Host "## Hit:" $hit
        $hit = "## URL: " + $line.Substring(5)
        write-Host $hit
        write-Host "=========================================================================="

    }
}

 
PowerShell
Edit|Remove
#
# RankLog parser configuration file used to set default parameters.
#
# Available parameters are
#  inputFile => search query file with ranklog (parsing existing files)
#  query     => FQL query
#  hits      => number of hits
#  sort      => sort criteria (rankprofile)
#  security  => Security claim token (found in query log file)
#            To use a security claim token from the script you need to:
#               1.	In the file $FASTSEARCH/components/sam/worker/user_config.xml set 
#              <add name="AllowNonCleanUpClaimsCacheForTestingOnly" value="false" type="System.Boolean" /> 
#               to 
#              <add name="AllowNonCleanUpClaimsCacheForTestingOnly" value="true" type="System.Boolean" />
#               2. Restart samworker (nctrl restart samworker)
#               3. Perform a query from SharePoint UI
#               4. Open query log and copy claim token from "qtf_securityfql:uid" parameter
#
inputFile=c:\temp\ranklog4.txt
query=azure
hits=20
sort=iprankprofile
security=MCMud3xyZWRtb25kXHYtbGlzYXNh