Uhm you don't sort it in anyway.
I use a strategy, get all the colours under a tolerance, use a for..to..do loop and create a 2D Point array of the occuraces of each colour. Use SortATPASize to sort this new atpa, go through checking all the colours and lengths of each occurance. Don't forget to use ClearSameIntegers 
Example:
SCAR Code:
function FindKaraTreeColour: Integer;
var
p: TPointArray;
a: T2DPointArray;
c: TIntegerArray;
i, h, cts: Integer;
R, G, B: Integer;
begin
Result := 0;
cts := GetColorToleranceSpeed;
ColorToleranceSpeed(1);
FindColorsTolerance(p, 3238985, MMX1, MMY1, MMX2, MMY2, 47);
FilterPointsPie(p, 0, 360, 10, 76, MMCX, MMCY);
if High(p) < 50 then
begin
ColorToleranceSpeed(cts);
Writeln('FindKaraTreeColour: Could not find Base Colour.');
Exit;
end;
c := GetColors(p);
ClearSameIntegers(c);
h := High(c);
SetLength(a, h + 1);
for i := 0 to h do
begin
ColorToRGB(c[i], R, G, B);
if (R >= 44) and (R <= 101) and (G >= 80) and (G <= 136) and (B >= 22) and (B <= 75) then
begin
FindColors(a[i], c[i], MMX1, MMY1, MMX2, MMY2);
FilterPointsPie(a[i], 0, 360, 10, 76, MMCX, MMCY);
end;
end;
SortATPASize(a, True);
Result := GetColor(a[0][0].x, a[0][0].y);
{$IFDEF DEBUG}Writeln('KaraTreeColour = ' + IntToStr(Result));{$ENDIF}
ColorToleranceSpeed(cts);
end;
Not the best example, but you can kinda see what I mean.