// Scintilla source code edit control
/** @file Document.cxx
 ** Text document that handles notifications, DBCS, styling, words and end of line.
 **/
// Copyright 1998-2011 by Neil Hodgson <neilh@scintilla.org>
// The License.txt file describes the conditions under which this software may be distributed.

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#include <assert.h>

#include <string>
#include <vector>

#include "Platform.h"

#include "ILexer.h"
#include "Scintilla.h"

#include "SplitVector.h"
#include "Partitioning.h"
#include "RunStyles.h"
#include "CellBuffer.h"
#include "PerLine.h"
#include "CharClassify.h"
#include "CharacterSet.h"
#include "Decoration.h"
#include "Document.h"
#include "RESearch.h"
#include "UniConversion.h"

#ifdef SCI_NAMESPACE
using namespace Scintilla;
#endif

// This is ASCII specific but is safe with chars >= 0x80
static inline bool isspacechar(unsigned char ch) {
        return (ch == ' ') || ((ch >= 0x09) && (ch <= 0x0d));
}

static inline bool IsPunctuation(char ch) {
        return isascii(ch) && ispunct(ch);
}

static inline bool IsADigit(char ch) {
        return isascii(ch) && isdigit(ch);
}

static inline bool IsLowerCase(char ch) {
        return isascii(ch) && islower(ch);
}

static inline bool IsUpperCase(char ch) {
        return isascii(ch) && isupper(ch);
}

void LexInterface::Colourise(int start, int end) {
        ElapsedTime et;
        if (pdoc && instance && !performingStyle) {
                // Protect against reentrance, which may occur, for example, when
                // fold points are discovered while performing styling and the folding
                // code looks for child lines which may trigger styling.
                performingStyle = true;

                int lengthDoc = pdoc->Length();
                if (end == -1)
                        end = lengthDoc;
                int len = end - start;

                PLATFORM_ASSERT(len >= 0);
                PLATFORM_ASSERT(start + len <= lengthDoc);

                int styleStart = 0;
                if (start > 0)
                        styleStart = pdoc->StyleAt(start - 1) & pdoc->stylingBitsMask;

                if (len > 0) {
                        instance->Lex(start, len, styleStart, pdoc);
                        instance->Fold(start, len, styleStart, pdoc);
                }

                performingStyle = false;
        }
}

Document::Document() {
        refCount = 0;
#ifdef _WIN32
        eolMode = SC_EOL_CRLF;
#else
        eolMode = SC_EOL_LF;
#endif
        dbcsCodePage = 0;
        stylingBits = 5;
        stylingBitsMask = 0x1F;
        stylingMask = 0;
        endStyled = 0;
        styleClock = 0;
        enteredModification = 0;
        enteredStyling = 0;
        enteredReadOnlyCount = 0;
        tabInChars = 8;
        indentInChars = 0;
        actualIndentInChars = 8;
        useTabs = true;
        tabIndents = true;
        backspaceUnindents = false;
        watchers = 0;
        lenWatchers = 0;

        matchesValid = false;
        regex = 0;

        perLineData[ldMarkers] = new LineMarkers();
        perLineData[ldLevels] = new LineLevels();
        perLineData[ldState] = new LineState();
        perLineData[ldMargin] = new LineAnnotation();
        perLineData[ldAnnotation] = new LineAnnotation();

        cb.SetPerLine(this);

        pli = 0;
}

Document::~Document() {
        for (int i = 0; i < lenWatchers; i++) {
                watchers[i].watcher->NotifyDeleted(this, watchers[i].userData);
        }
        delete []watchers;
        for (int j=0; j<ldSize; j++) {
                delete perLineData[j];
                perLineData[j] = 0;
        }
        watchers = 0;
        lenWatchers = 0;
        delete regex;
        regex = 0;
        delete pli;
        pli = 0;
}

void Document::Init() {
        for (int j=0; j<ldSize; j++) {
                if (perLineData[j])
                        perLineData[j]->Init();
        }
}

void Document::InsertLine(int line) {
        for (int j=0; j<ldSize; j++) {
                if (perLineData[j])
                        perLineData[j]->InsertLine(line);
        }
}

void Document::RemoveLine(int line) {
        for (int j=0; j<ldSize; j++) {
                if (perLineData[j])
                        perLineData[j]->RemoveLine(line);
        }
}

// Increase reference count and return its previous value.
int Document::AddRef() {
        return refCount++;
}

// Decrease reference count and return its previous value.
// Delete the document if reference count reaches zero.
int SCI_METHOD Document::Release() {
        int curRefCount = --refCount;
        if (curRefCount == 0)
                delete this;
        return curRefCount;
}

void Document::SetSavePoint() {
        cb.SetSavePoint();
        NotifySavePoint(true);
}

int Document::GetMark(int line) {
        return static_cast<LineMarkers *>(perLineData[ldMarkers])->MarkValue(line);
}

int Document::MarkerNext(int lineStart, int mask) const {
        return static_cast<LineMarkers *>(perLineData[ldMarkers])->MarkerNext(lineStart, mask);
}

int Document::AddMark(int line, int markerNum) {
        if (line >= 0 && line <= LinesTotal()) {
                int prev = static_cast<LineMarkers *>(perLineData[ldMarkers])->
                        AddMark(line, markerNum, LinesTotal());
                DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, 0, line);
                NotifyModified(mh);
                return prev;
        } else {
                return 0;
        }
}

void Document::AddMarkSet(int line, int valueSet) {
        if (line < 0 || line > LinesTotal()) {
                return;
        }
        unsigned int m = valueSet;
        for (int i = 0; m; i++, m >>= 1)
                if (m & 1)
                        static_cast<LineMarkers *>(perLineData[ldMarkers])->
                                AddMark(line, i, LinesTotal());
        DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, 0, line);
        NotifyModified(mh);
}

void Document::DeleteMark(int line, int markerNum) {
        static_cast<LineMarkers *>(perLineData[ldMarkers])->DeleteMark(line, markerNum, false);
        DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, 0, line);
        NotifyModified(mh);
}

void Document::DeleteMarkFromHandle(int markerHandle) {
        static_cast<LineMarkers *>(perLineData[ldMarkers])->DeleteMarkFromHandle(markerHandle);
        DocModification mh(SC_MOD_CHANGEMARKER, 0, 0, 0, 0);
        mh.line = -1;
        NotifyModified(mh);
}

void Document::DeleteAllMarks(int markerNum) {
        bool someChanges = false;
        for (int line = 0; line < LinesTotal(); line++) {
                if (static_cast<LineMarkers *>(perLineData[ldMarkers])->DeleteMark(line, markerNum, true))
                        someChanges = true;
        }
        if (someChanges) {
                DocModification mh(SC_MOD_CHANGEMARKER, 0, 0, 0, 0);
                mh.line = -1;
                NotifyModified(mh);
        }
}

int Document::LineFromHandle(int markerHandle) {
        return static_cast<LineMarkers *>(perLineData[ldMarkers])->LineFromHandle(markerHandle);
}

int SCI_METHOD Document::LineStart(int line) const {
        return cb.LineStart(line);
}

int Document::LineEnd(int line) const {
        if (line == LinesTotal() - 1) {
                return LineStart(line + 1);
        } else {
                int position = LineStart(line + 1) - 1;
                // When line terminator is CR+LF, may need to go back one more
                if ((position > LineStart(line)) && (cb.CharAt(position - 1) == '\r')) {
                        position--;
                }
                return position;
        }
}

void SCI_METHOD Document::SetErrorStatus(int status) {
        // Tell the watchers the lexer has changed.
        for (int i = 0; i < lenWatchers; i++) {
                watchers[i].watcher->NotifyErrorOccurred(this, watchers[i].userData, status);
        }
}

int SCI_METHOD Document::LineFromPosition(int pos) const {
        return cb.LineFromPosition(pos);
}

int Document::LineEndPosition(int position) const {
        return LineEnd(LineFromPosition(position));
}

bool Document::IsLineEndPosition(int position) const {
        return LineEnd(LineFromPosition(position)) == position;
}

int Document::VCHomePosition(int position) const {
        int line = LineFromPosition(position);
        int startPosition = LineStart(line);
        int endLine = LineEnd(line);
        int startText = startPosition;
        while (startText < endLine && (cb.CharAt(startText) == ' ' || cb.CharAt(startText) == '\t'))
                startText++;
        if (position == startText)
                return startPosition;
        else
                return startText;
}

int SCI_METHOD Document::SetLevel(int line, int level) {
        int prev = static_cast<LineLevels *>(perLineData[ldLevels])->SetLevel(line, level, LinesTotal());
        if (prev != level) {
                DocModification mh(SC_MOD_CHANGEFOLD | SC_MOD_CHANGEMARKER,
                                   LineStart(line), 0, 0, 0, line);
                mh.foldLevelNow = level;
                mh.foldLevelPrev = prev;
                NotifyModified(mh);
        }
        return prev;
}

int SCI_METHOD Document::GetLevel(int line) const {
        return static_cast<LineLevels *>(perLineData[ldLevels])->GetLevel(line);
}

void Document::ClearLevels() {
        static_cast<LineLevels *>(perLineData[ldLevels])->ClearLevels();
}

static bool IsSubordinate(int levelStart, int levelTry) {
        if (levelTry & SC_FOLDLEVELWHITEFLAG)
                return true;
        else
                return (levelStart & SC_FOLDLEVELNUMBERMASK) < (levelTry & SC_FOLDLEVELNUMBERMASK);
}

int Document::GetLastChild(int lineParent, int level, int lastLine) {
        if (level == -1)
                level = GetLevel(lineParent) & SC_FOLDLEVELNUMBERMASK;
        int maxLine = LinesTotal();
        int lookLastLine = (lastLine != -1) ? Platform::Minimum(LinesTotal() - 1, lastLine) : -1;
        int lineMaxSubord = lineParent;
        while (lineMaxSubord < maxLine - 1) {
                EnsureStyledTo(LineStart(lineMaxSubord + 2));
                if (!IsSubordinate(level, GetLevel(lineMaxSubord + 1)))
                        break;
                if ((lookLastLine != -1) && (lineMaxSubord >= lookLastLine) && !(GetLevel(lineMaxSubord) & SC_FOLDLEVELWHITEFLAG))
                        break;
                lineMaxSubord++;
        }
        if (lineMaxSubord > lineParent) {
                if (level > (GetLevel(lineMaxSubord + 1) & SC_FOLDLEVELNUMBERMASK)) {
                        // Have chewed up some whitespace that belongs to a parent so seek back
                        if (GetLevel(lineMaxSubord) & SC_FOLDLEVELWHITEFLAG) {
                                lineMaxSubord--;
                        }
                }
        }
        return lineMaxSubord;
}

int Document::GetFoldParent(int line) {
        int level = GetLevel(line) & SC_FOLDLEVELNUMBERMASK;
        int lineLook = line - 1;
        while ((lineLook > 0) && (
                    (!(GetLevel(lineLook) & SC_FOLDLEVELHEADERFLAG)) ||
                    ((GetLevel(lineLook) & SC_FOLDLEVELNUMBERMASK) >= level))
              ) {
                lineLook--;
        }
        if ((GetLevel(lineLook) & SC_FOLDLEVELHEADERFLAG) &&
                ((GetLevel(lineLook) & SC_FOLDLEVELNUMBERMASK) < level)) {
                return lineLook;
        } else {
                return -1;
        }
}

void Document::GetHighlightDelimiters(HighlightDelimiter &highlightDelimiter, int line, int lastLine) {
        int level = GetLevel(line);
        int lookLastLine = Platform::Maximum(line, lastLine) + 1;

        int lookLine = line;
        int lookLineLevel = level;
        int lookLineLevelNum = lookLineLevel & SC_FOLDLEVELNUMBERMASK;
        while ((lookLine > 0) && ((lookLineLevel & SC_FOLDLEVELWHITEFLAG) || 
                ((lookLineLevel & SC_FOLDLEVELHEADERFLAG) && (lookLineLevelNum >= (GetLevel(lookLine + 1) & SC_FOLDLEVELNUMBERMASK))))) {
                lookLineLevel = GetLevel(--lookLine);
                lookLineLevelNum = lookLineLevel & SC_FOLDLEVELNUMBERMASK;
        }

        int beginFoldBlock = (lookLineLevel & SC_FOLDLEVELHEADERFLAG) ? lookLine : GetFoldParent(lookLine);
        if (beginFoldBlock == -1) {
                highlightDelimiter.Clear();
                return;
        }

        int endFoldBlock = GetLastChild(beginFoldBlock, -1, lookLastLine);
        int firstChangeableLineBefore = -1;
        if (endFoldBlock < line) {
                lookLine = beginFoldBlock - 1;
                lookLineLevel = GetLevel(lookLine);
                lookLineLevelNum = lookLineLevel & SC_FOLDLEVELNUMBERMASK;
                while ((lookLine >= 0) && (lookLineLevelNum >= SC_FOLDLEVELBASE)) {
                        if (lookLineLevel & SC_FOLDLEVELHEADERFLAG) {
                                if (GetLastChild(lookLine, -1, lookLastLine) == line) {
                                        beginFoldBlock = lookLine;
                                        endFoldBlock = line;
                                        firstChangeableLineBefore = line - 1;
                                }
                        }
                        if ((lookLine > 0) && (lookLineLevelNum == SC_FOLDLEVELBASE) && ((GetLevel(lookLine - 1) & SC_FOLDLEVELNUMBERMASK) > lookLineLevelNum))
                                break;
                        lookLineLevel = GetLevel(--lookLine);
                        lookLineLevelNum = lookLineLevel & SC_FOLDLEVELNUMBERMASK;
                }
        }
        if (firstChangeableLineBefore == -1) {
                for (lookLine = line - 1, lookLineLevel = GetLevel(lookLine), lookLineLevelNum = lookLineLevel & SC_FOLDLEVELNUMBERMASK; 
                        lookLine >= beginFoldBlock; 
                        lookLineLevel = GetLevel(--lookLine), lookLineLevelNum = lookLineLevel & SC_FOLDLEVELNUMBERMASK) {
                        if ((lookLineLevel & SC_FOLDLEVELWHITEFLAG) || (lookLineLevelNum > (level & SC_FOLDLEVELNUMBERMASK))) {
                                firstChangeableLineBefore = lookLine;
                                break;
                        }
                }
        }
        if (firstChangeableLineBefore == -1)
                firstChangeableLineBefore = beginFoldBlock - 1;

        int firstChangeableLineAfter = -1;
        for (lookLine = line + 1, lookLineLevel = GetLevel(lookLine), lookLineLevelNum = lookLineLevel & SC_FOLDLEVELNUMBERMASK; 
                lookLine <= endFoldBlock; 
                lookLineLevel = GetLevel(++lookLine), lookLineLevelNum = lookLineLevel & SC_FOLDLEVELNUMBERMASK) {
                if ((lookLineLevel & SC_FOLDLEVELHEADERFLAG) && (lookLineLevelNum < (GetLevel(lookLine + 1) & SC_FOLDLEVELNUMBERMASK))) {
                        firstChangeableLineAfter = lookLine;
                        break;
                }
        }
        if (firstChangeableLineAfter == -1)
                firstChangeableLineAfter = endFoldBlock + 1;

        highlightDelimiter.beginFoldBlock = beginFoldBlock;
        highlightDelimiter.endFoldBlock = endFoldBlock;
        highlightDelimiter.firstChangeableLineBefore = firstChangeableLineBefore;
        highlightDelimiter.firstChangeableLineAfter = firstChangeableLineAfter;
}

int Document::ClampPositionIntoDocument(int pos) {
        return Platform::Clamp(pos, 0, Length());
}

bool Document::IsCrLf(int pos) {
        if (pos < 0)
                return false;
        if (pos >= (Length() - 1))
                return false;
        return (cb.CharAt(pos) == '\r') && (cb.CharAt(pos + 1) == '\n');
}

int Document::LenChar(int pos) {
        if (pos < 0) {
                return 1;
        } else if (IsCrLf(pos)) {
                return 2;
        } else if (SC_CP_UTF8 == dbcsCodePage) {
                unsigned char ch = static_cast<unsigned char>(cb.CharAt(pos));
                if (ch < 0x80)
                        return 1;
                int len = 2;
                if (ch >= (0x80 + 0x40 + 0x20 + 0x10))
                        len = 4;
                else if (ch >= (0x80 + 0x40 + 0x20))
                        len = 3;
                int lengthDoc = Length();
                if ((pos + len) > lengthDoc)
                        return lengthDoc -pos;
                else
                        return len;
        } else if (dbcsCodePage) {
                return IsDBCSLeadByte(cb.CharAt(pos)) ? 2 : 1;
        } else {
                return 1;
        }
}

static bool IsTrailByte(int ch) {
        return (ch >= 0x80) && (ch < (0x80 + 0x40));
}

static int BytesFromLead(int leadByte) {
        if (leadByte > 0xF4) {
                // Characters longer than 4 bytes not possible in current UTF-8
                return 0;
        } else if (leadByte >= 0xF0) {
                return 4;
        } else if (leadByte >= 0xE0) {
                return 3;
        } else if (leadByte >= 0xC2) {
                return 2;
        }
        return 0;
}

bool Document::InGoodUTF8(int pos, int &start, int &end) const {
        int lead = pos;
        while ((lead>0) && (pos-lead < 4) && IsTrailByte(static_cast<unsigned char>(cb.CharAt(lead-1))))
                lead--;
        start = 0;
        if (lead > 0) {
                start = lead-1;
        }
        int leadByte = static_cast<unsigned char>(cb.CharAt(start));
        int bytes = BytesFromLead(leadByte);
        if (bytes == 0) {
                return false;
        } else {
                int trailBytes = bytes - 1;
                int len = pos - lead + 1;
                if (len > trailBytes)
                        // pos too far from lead
                        return false;
                // Check that there are enough trails for this lead
                int trail = pos + 1;
                while ((trail-lead<trailBytes) && (trail < Length())) {
                        if (!IsTrailByte(static_cast<unsigned char>(cb.CharAt(trail)))) {
                                return false;
                        }
                        trail++;
                }
                end = start + bytes;
                return true;
        }
}

// Normalise a position so that it is not halfway through a two byte character.
// This can occur in two situations -
// When lines are terminated with \r\n pairs which should be treated as one character.
// When displaying DBCS text such as Japanese.
// If moving, move the position in the indicated direction.
int Document::MovePositionOutsideChar(int pos, int moveDir, bool checkLineEnd) {
        //Platform::DebugPrintf("NoCRLF %d %d\n", pos, moveDir);
        // If out of range, just return minimum/maximum value.
        if (pos <= 0)
                return 0;
        if (pos >= Length())
                return Length();

        // PLATFORM_ASSERT(pos > 0 && pos < Length());
        if (checkLineEnd && IsCrLf(pos - 1)) {
                if (moveDir > 0)
                        return pos + 1;
                else
                        return pos - 1;
        }

        if (dbcsCodePage) {
                if (SC_CP_UTF8 == dbcsCodePage) {
                        unsigned char ch = static_cast<unsigned char>(cb.CharAt(pos));
                        int startUTF = pos;
                        int endUTF = pos;
                        if (IsTrailByte(ch) && InGoodUTF8(pos, startUTF, endUTF)) {
                                // ch is a trail byte within a UTF-8 character
                                if (moveDir > 0)
                                        pos = endUTF;
                                else
                                        pos = startUTF;
                        }
                } else {
                        // Anchor DBCS calculations at start of line because start of line can
                        // not be a DBCS trail byte.
                        int posStartLine = LineStart(LineFromPosition(pos));
                        if (pos == posStartLine)
                                return pos;

                        // Step back until a non-lead-byte is found.
                        int posCheck = pos;
                        while ((posCheck > posStartLine) && IsDBCSLeadByte(cb.CharAt(posCheck-1)))
                                posCheck--;

                        // Check from known start of character.
                        while (posCheck < pos) {
                                int mbsize = IsDBCSLeadByte(cb.CharAt(posCheck)) ? 2 : 1;
                                if (posCheck + mbsize == pos) {
                                        return pos;
                                } else if (posCheck + mbsize > pos) {
                                        if (moveDir > 0) {
                                                return posCheck + mbsize;
                                        } else {
                                                return posCheck;
                                        }
                                }
                                posCheck += mbsize;
                        }
                }
        }

        return pos;
}

// NextPosition moves between valid positions - it can not handle a position in the middle of a
// multi-byte character. It is used to iterate through text more efficiently than MovePositionOutsideChar.
// A \r\n pair is treated as two characters.
int Document::NextPosition(int pos, int moveDir) const {
        // If out of range, just return minimum/maximum value.
        int increment = (moveDir > 0) ? 1 : -1;
        if (pos + increment <= 0)
                return 0;
        if (pos + increment >= Length())
                return Length();

        if (dbcsCodePage) {
                if (SC_CP_UTF8 == dbcsCodePage) {
                        pos += increment;
                        unsigned char ch = static_cast<unsigned char>(cb.CharAt(pos));
                        int startUTF = pos;
                        int endUTF = pos;
                        if (IsTrailByte(ch) && InGoodUTF8(pos, startUTF, endUTF)) {
                                // ch is a trail byte within a UTF-8 character
                                if (moveDir > 0)
                                        pos = endUTF;
                                else
                                        pos = startUTF;
                        }
                } else {
                        if (moveDir > 0) {
                                int mbsize = IsDBCSLeadByte(cb.CharAt(pos)) ? 2 : 1;
                                pos += mbsize;
                                if (pos > Length())
                                        pos = Length();
                        } else {
                                // Anchor DBCS calculations at start of line because start of line can
                                // not be a DBCS trail byte.
                                int posStartLine = LineStart(LineFromPosition(pos));
                                // See http://msdn.microsoft.com/en-us/library/cc194792%28v=MSDN.10%29.aspx
                                // http://msdn.microsoft.com/en-us/library/cc194790.aspx
                                if ((pos - 1) <= posStartLine) {
                                        return pos - 1;
                                } else if (IsDBCSLeadByte(cb.CharAt(pos - 1))) {
                                        // Must actually be trail byte
                                        return pos - 2;
                                } else {
                                        // Otherwise, step back until a non-lead-byte is found.
                                        int posTemp = pos - 1;
                                        while (posStartLine <= --posTemp && IsDBCSLeadByte(cb.CharAt(posTemp)))
                                                ;
                                        // Now posTemp+1 must point to the beginning of a character,
                                        // so figure out whether we went back an even or an odd
                                        // number of bytes and go back 1 or 2 bytes, respectively.
                                        return (pos - 1 - ((pos - posTemp) & 1));
                                }
                        }
                }
        } else {
                pos += increment;
        }

        return pos;
}

bool Document::NextCharacter(int &pos, int moveDir) {
        // Returns true if pos changed
        int posNext = NextPosition(pos, moveDir);
        if (posNext == pos) {
                return false;
        } else {
                pos = posNext;
                return true;
        }
}

int SCI_METHOD Document::CodePage() const {
        return dbcsCodePage;
}

bool SCI_METHOD Document::IsDBCSLeadByte(char ch) const {
        // Byte ranges found in Wikipedia articles with relevant search strings in each case
        unsigned char uch = static_cast<unsigned char>(ch);
        switch (dbcsCodePage) {
                case 932:
                        // Shift_jis
                        return ((uch >= 0x81) && (uch <= 0x9F)) ||
                                ((uch >= 0xE0) && (uch <= 0xFC));
                                // Lead bytes F0 to FC may be a Microsoft addition. 
                case 936:
                        // GBK
                        return (uch >= 0x81) && (uch <= 0xFE);
                case 949:
                        // Korean Wansung KS C-5601-1987
                        return (uch >= 0x81) && (uch <= 0xFE);
                case 950:
                        // Big5
                        return (uch >= 0x81) && (uch <= 0xFE);
                case 1361:
                        // Korean Johab KS C-5601-1992
                        return
                                ((uch >= 0x84) && (uch <= 0xD3)) ||
                                ((uch >= 0xD8) && (uch <= 0xDE)) ||
                                ((uch >= 0xE0) && (uch <= 0xF9));
        }
        return false;
}

inline bool IsSpaceOrTab(int ch) {
        return ch == ' ' || ch == '\t';
}

// Need to break text into segments near lengthSegment but taking into
// account the encoding to not break inside a UTF-8 or DBCS character
// and also trying to avoid breaking inside a pair of combining characters.
// The segment length must always be long enough (more than 4 bytes)
// so that there will be at least one whole character to make a segment.
// For UTF-8, text must consist only of valid whole characters.
// In preference order from best to worst:
//   1) Break after space
//   2) Break before punctuation
//   3) Break after whole character

int Document::SafeSegment(const char *text, int length, int lengthSegment) {
        if (length <= lengthSegment)
                return length;
        int lastSpaceBreak = -1;
        int lastPunctuationBreak = -1;
        int lastEncodingAllowedBreak = -1;
        for (int j=0; j < lengthSegment;) {
                unsigned char ch = static_cast<unsigned char>(text[j]);
                if (j > 0) {
                        if (IsSpaceOrTab(text[j - 1]) && !IsSpaceOrTab(text[j])) {
                                lastSpaceBreak = j;
                        }
                        if (ch < 'A') {
                                lastPunctuationBreak = j;
                        }
                }
                lastEncodingAllowedBreak = j;

                if (dbcsCodePage == SC_CP_UTF8) {
                        j += (ch < 0x80) ? 1 : BytesFromLead(ch);
                } else if (dbcsCodePage) {
                        j += IsDBCSLeadByte(ch) ? 2 : 1;
                } else {
                        j++;
                }
        }
        if (lastSpaceBreak >= 0) {
                return lastSpaceBreak;
        } else if (lastPunctuationBreak >= 0) {
                return lastPunctuationBreak;
        }
        return lastEncodingAllowedBreak;
}

void Document::ModifiedAt(int pos) {
        if (endStyled > pos)
                endStyled = pos;
}

void Document::CheckReadOnly() {
        if (cb.IsReadOnly() && enteredReadOnlyCount == 0) {
                enteredReadOnlyCount++;
                NotifyModifyAttempt();
                enteredReadOnlyCount--;
        }
}

// Document only modified by gateways DeleteChars, InsertString, Undo, Redo, and SetStyleAt.
// SetStyleAt does not change the persistent state of a document

bool Document::DeleteChars(int pos, int len) {
        if (len == 0)
                return false;
        if ((pos + len) > Length())
                return false;
        CheckReadOnly();
        if (enteredModification != 0) {
                return false;
        } else {
                enteredModification++;
                if (!cb.IsReadOnly()) {
                        NotifyModified(
                            DocModification(
                                SC_MOD_BEFOREDELETE | SC_PERFORMED_USER,
                                pos, len,
                                0, 0));
                        int prevLinesTotal = LinesTotal();
                        bool startSavePoint = cb.IsSavePoint();
                        bool startSequence = false;
                        const char *text = cb.DeleteChars(pos, len, startSequence);
                        if (startSavePoint && cb.IsCollectingUndo())
                                NotifySavePoint(!startSavePoint);
                        if ((pos < Length()) || (pos == 0))
                                ModifiedAt(pos);
                        else
                                ModifiedAt(pos-1);
                        NotifyModified(
                            DocModification(
                                SC_MOD_DELETETEXT | SC_PERFORMED_USER | (startSequence?SC_STARTACTION:0),
                                pos, len,
                                LinesTotal() - prevLinesTotal, text));
                }
                enteredModification--;
        }
        return !cb.IsReadOnly();
}

/**
 * Insert a string with a length.
 */
bool Document::InsertString(int position, const char *s, int insertLength) {
        if (insertLength <= 0) {
                return false;
        }
        CheckReadOnly();
        if (enteredModification != 0) {
                return false;
        } else {
                enteredModification++;
                if (!cb.IsReadOnly()) {
                        NotifyModified(
                            DocModification(
                                SC_MOD_BEFOREINSERT | SC_PERFORMED_USER,
                                position, insertLength,
                                0, s));
                        int prevLinesTotal = LinesTotal();
                        bool startSavePoint = cb.IsSavePoint();
                        bool startSequence = false;
                        const char *text = cb.InsertString(position, s, insertLength, startSequence);
                        if (startSavePoint && cb.IsCollectingUndo())
                                NotifySavePoint(!startSavePoint);
                        ModifiedAt(position);
                        NotifyModified(
                            DocModification(
                                SC_MOD_INSERTTEXT | SC_PERFORMED_USER | (startSequence?SC_STARTACTION:0),
                                position, insertLength,
                                LinesTotal() - prevLinesTotal, text));
                }
                enteredModification--;
        }
        return !cb.IsReadOnly();
}

int SCI_METHOD Document::AddData(char *data, int length) {
        try {
                int position = Length();
                InsertString(position,data, length);
        } catch (std::bad_alloc &) {
                return SC_STATUS_BADALLOC;
        } catch (...) {
                return SC_STATUS_FAILURE;
        }
        return 0;
}

void * SCI_METHOD Document::ConvertToDocument() {
        return this;
}

int Document::Undo() {
        int newPos = -1;
        CheckReadOnly();
        if (enteredModification == 0) {
                enteredModification++;
                if (!cb.IsReadOnly()) {
                        bool startSavePoint = cb.IsSavePoint();
                        bool multiLine = false;
                        int steps = cb.StartUndo();
                        //Platform::DebugPrintf("Steps=%d\n", steps);
                        for (int step = 0; step < steps; step++) {
                                const int prevLinesTotal = LinesTotal();
                                const Action &action = cb.GetUndoStep();
                                if (action.at == removeAction) {
                                        NotifyModified(DocModification(
                                                                        SC_MOD_BEFOREINSERT | SC_PERFORMED_UNDO, action));
                                } else if (action.at == containerAction) {
                                        DocModification dm(SC_MOD_CONTAINER | SC_PERFORMED_UNDO);
                                        dm.token = action.position;
                                        NotifyModified(dm);
                                } else {
                                        NotifyModified(DocModification(
                                                                        SC_MOD_BEFOREDELETE | SC_PERFORMED_UNDO, action));
                                }
                                cb.PerformUndoStep();
                                int cellPosition = action.position;
                                if (action.at != containerAction) {
                                        ModifiedAt(cellPosition);
                                        newPos = cellPosition;
                                }

                                int modFlags = SC_PERFORMED_UNDO;
                                // With undo, an insertion action becomes a deletion notification
                                if (action.at == removeAction) {
                                        newPos += action.lenData;
                                        modFlags |= SC_MOD_INSERTTEXT;
                                } else if (action.at == insertAction) {
                                        modFlags |= SC_MOD_DELETETEXT;
                                }
                                if (steps > 1)
                                        modFlags |= SC_MULTISTEPUNDOREDO;
                                const int linesAdded = LinesTotal() - prevLinesTotal;
                                if (linesAdded != 0)
                                        multiLine = true;
                                if (step == steps - 1) {
                                        modFlags |= SC_LASTSTEPINUNDOREDO;
                                        if (multiLine)
                                                modFlags |= SC_MULTILINEUNDOREDO;
                                }
                                NotifyModified(DocModification(modFlags, cellPosition, action.lenData,
                                                                                           linesAdded, action.data));
                        }

                        bool endSavePoint = cb.IsSavePoint();
                        if (startSavePoint != endSavePoint)
                                NotifySavePoint(endSavePoint);
                }
                enteredModification--;
        }
        return newPos;
}

int Document::Redo() {
        int newPos = -1;
        CheckReadOnly();
        if (enteredModification == 0) {
                enteredModification++;
                if (!cb.IsReadOnly()) {
                        bool startSavePoint = cb.IsSavePoint();
                        bool multiLine = false;
                        int steps = cb.StartRedo();
                        for (int step = 0; step < steps; step++) {
                                const int prevLinesTotal = LinesTotal();
                                const Action &action = cb.GetRedoStep();
                                if (action.at == insertAction) {
                                        NotifyModified(DocModification(
                                                                        SC_MOD_BEFOREINSERT | SC_PERFORMED_REDO, action));
                                } else if (action.at == containerAction) {
                                        DocModification dm(SC_MOD_CONTAINER | SC_PERFORMED_REDO);
                                        dm.token = action.position;
                                        NotifyModified(dm);
                                } else {
                                        NotifyModified(DocModification(
                                                                        SC_MOD_BEFOREDELETE | SC_PERFORMED_REDO, action));
                                }
                                cb.PerformRedoStep();
                                if (action.at != containerAction) {
                                        ModifiedAt(action.position);
                                        newPos = action.position;
                                }

                                int modFlags = SC_PERFORMED_REDO;
                                if (action.at == insertAction) {
                                        newPos += action.lenData;
                                        modFlags |= SC_MOD_INSERTTEXT;
                                } else if (action.at == removeAction) {
                                        modFlags |= SC_MOD_DELETETEXT;
                                }
                                if (steps > 1)
                                        modFlags |= SC_MULTISTEPUNDOREDO;
                                const int linesAdded = LinesTotal() - prevLinesTotal;
                                if (linesAdded != 0)
                                        multiLine = true;
                                if (step == steps - 1) {
                                        modFlags |= SC_LASTSTEPINUNDOREDO;
                                        if (multiLine)
                                                modFlags |= SC_MULTILINEUNDOREDO;
                                }
                                NotifyModified(
                                        DocModification(modFlags, action.position, action.lenData,
                                                                        linesAdded, action.data));
                        }

                        bool endSavePoint = cb.IsSavePoint();
                        if (startSavePoint != endSavePoint)
                                NotifySavePoint(endSavePoint);
                }
                enteredModification--;
        }
        return newPos;
}

/**
 * Insert a single character.
 */
bool Document::InsertChar(int pos, char ch) {
        char chs[1];
        chs[0] = ch;
        return InsertString(pos, chs, 1);
}

/**
 * Insert a null terminated string.
 */
bool Document::InsertCString(int position, const char *s) {
        return InsertString(position, s, static_cast<int>(strlen(s)));
}

void Document::ChangeChar(int pos, char ch) {
        DeleteChars(pos, 1);
        InsertChar(pos, ch);
}

void Document::DelChar(int pos) {
        DeleteChars(pos, LenChar(pos));
}

void Document::DelCharBack(int pos) {
        if (pos <= 0) {
                return;
        } else if (IsCrLf(pos - 2)) {
                DeleteChars(pos - 2, 2);
        } else if (dbcsCodePage) {
                int startChar = NextPosition(pos, -1);
                DeleteChars(startChar, pos - startChar);
        } else {
                DeleteChars(pos - 1, 1);
        }
}

static bool isindentchar(char ch) {
        return (ch == ' ') || (ch == '\t');
}

static int NextTab(int pos, int tabSize) {
        return ((pos / tabSize) + 1) * tabSize;
}

static void CreateIndentation(char *linebuf, int length, int indent, int tabSize, bool insertSpaces) {
        length--;       // ensure space for \0
        if (!insertSpaces) {
                while ((indent >= tabSize) && (length > 0)) {
                        *linebuf++ = '\t';
                        indent -= tabSize;
                        length--;
                }
        }
        while ((indent > 0) && (length > 0)) {
                *linebuf++ = ' ';
                indent--;
                length--;
        }
        *linebuf = '\0';
}

int SCI_METHOD Document::GetLineIndentation(int line) {
        int indent = 0;
        if ((line >= 0) && (line < LinesTotal())) {
                int lineStart = LineStart(line);
                int length = Length();
                for (int i = lineStart; i < length; i++) {
                        char ch = cb.CharAt(i);
                        if (ch == ' ')
                                indent++;
                        else if (ch == '\t')
                                indent = NextTab(indent, tabInChars);
                        else
                                return indent;
                }
        }
        return indent;
}

void Document::SetLineIndentation(int line, int indent) {
        int indentOfLine = GetLineIndentation(line);
        if (indent < 0)
                indent = 0;
        if (indent != indentOfLine) {
                char linebuf[1000];
                CreateIndentation(linebuf, sizeof(linebuf), indent, tabInChars, !useTabs);
                int thisLineStart = LineStart(line);
                int indentPos = GetLineIndentPosition(line);
                UndoGroup ug(this);
                DeleteChars(thisLineStart, indentPos - thisLineStart);
                InsertCString(thisLineStart, linebuf);
        }
}

int Document::GetLineIndentPosition(int line) const {
        if (line < 0)
                return 0;
        int pos = LineStart(line);
        int length = Length();
        while ((pos < length) && isindentchar(cb.CharAt(pos))) {
                pos++;
        }
        return pos;
}

int Document::GetColumn(int pos) {
        int column = 0;
        int line = LineFromPosition(pos);
        if ((line >= 0) && (line < LinesTotal())) {
                for (int i = LineStart(line); i < pos;) {
                        char ch = cb.CharAt(i);
                        if (ch == '\t') {
                                column = NextTab(column, tabInChars);
                                i++;
                        } else if (ch == '\r') {
                                return column;
                        } else if (ch == '\n') {
                                return column;
                        } else if (i >= Length()) {
                                return column;
                        } else {
                                column++;
                                i = NextPosition(i, 1);
                        }
                }
        }
        return column;
}

int Document::CountCharacters(int startPos, int endPos) {
        startPos = MovePositionOutsideChar(startPos, 1, false);
        endPos = MovePositionOutsideChar(endPos, -1, false);
        int count = 0;
        int i = startPos;
        while (i < endPos) {
                count++;
                if (IsCrLf(i))
                        i++;
                i = NextPosition(i, 1);
        }
        return count;
}

int Document::FindColumn(int line, int column) {
        int position = LineStart(line);
        if ((line >= 0) && (line < LinesTotal())) {
                int columnCurrent = 0;
                while ((columnCurrent < column) && (position < Length())) {
                        char ch = cb.CharAt(position);
                        if (ch == '\t') {
                                columnCurrent = NextTab(columnCurrent, tabInChars);
                                position++;
                        } else if (ch == '\r') {
                                return position;
                        } else if (ch == '\n') {
                                return position;
                        } else {
                                columnCurrent++;
                                position = NextPosition(position, 1);
                        }
                }
        }
        return position;
}

void Document::Indent(bool forwards, int lineBottom, int lineTop) {
        // Dedent - suck white space off the front of the line to dedent by equivalent of a tab
        for (int line = lineBottom; line >= lineTop; line--) {
                int indentOfLine = GetLineIndentation(line);
                if (forwards) {
                        if (LineStart(line) < LineEnd(line)) {
                                SetLineIndentation(line, indentOfLine + IndentSize());
                        }
                } else {
                        SetLineIndentation(line, indentOfLine - IndentSize());
                }
        }
}

// Convert line endings for a piece of text to a particular mode.
// Stop at len or when a NUL is found.
// Caller must delete the returned pointer.
char *Document::TransformLineEnds(int *pLenOut, const char *s, size_t len, int eolModeWanted) {
        char *dest = new char[2 * len + 1];
        const char *sptr = s;
        char *dptr = dest;
        for (size_t i = 0; (i < len) && (*sptr != '\0'); i++) {
                if (*sptr == '\n' || *sptr == '\r') {
                        if (eolModeWanted == SC_EOL_CR) {
                                *dptr++ = '\r';
                        } else if (eolModeWanted == SC_EOL_LF) {
                                *dptr++ = '\n';
                        } else { // eolModeWanted == SC_EOL_CRLF
                                *dptr++ = '\r';
                                *dptr++ = '\n';
                        }
                        if ((*sptr == '\r') && (i+1 < len) && (*(sptr+1) == '\n')) {
                                i++;
                                sptr++;
                        }
                        sptr++;
                } else {
                        *dptr++ = *sptr++;
                }
        }
        *dptr++ = '\0';
        *pLenOut = (dptr - dest) - 1;
        return dest;
}

void Document::ConvertLineEnds(int eolModeSet) {
        UndoGroup ug(this);

        for (int pos = 0; pos < Length(); pos++) {
                if (cb.CharAt(pos) == '\r') {
                        if (cb.CharAt(pos + 1) == '\n') {
                                // CRLF
                                if (eolModeSet == SC_EOL_CR) {
                                        DeleteChars(pos + 1, 1); // Delete the LF
                                } else if (eolModeSet == SC_EOL_LF) {
                                        DeleteChars(pos, 1); // Delete the CR
                                } else {
                                        pos++;
                                }
                        } else {
                                // CR
                                if (eolModeSet == SC_EOL_CRLF) {
                                        InsertString(pos + 1, "\n", 1); // Insert LF
                                        pos++;
                                } else if (eolModeSet == SC_EOL_LF) {
                                        InsertString(pos, "\n", 1); // Insert LF
                                        DeleteChars(pos + 1, 1); // Delete CR
                                }
                        }
                } else if (cb.CharAt(pos) == '\n') {
                        // LF
                        if (eolModeSet == SC_EOL_CRLF) {
                                InsertString(pos, "\r", 1); // Insert CR
                                pos++;
                        } else if (eolModeSet == SC_EOL_CR) {
                                InsertString(pos, "\r", 1); // Insert CR
                                DeleteChars(pos + 1, 1); // Delete LF
                        }
                }
        }

}

bool Document::IsWhiteLine(int line) const {
        int currentChar = LineStart(line);
        int endLine = LineEnd(line);
        while (currentChar < endLine) {
                if (cb.CharAt(currentChar) != ' ' && cb.CharAt(currentChar) != '\t') {
                        return false;
                }
                ++currentChar;
        }
        return true;
}

int Document::ParaUp(int pos) {
        int line = LineFromPosition(pos);
        line--;
        while (line >= 0 && IsWhiteLine(line)) { // skip empty lines
                line--;
        }
        while (line >= 0 && !IsWhiteLine(line)) { // skip non-empty lines
                line--;
        }
        line++;
        return LineStart(line);
}

int Document::ParaDown(int pos) {
        int line = LineFromPosition(pos);
        while (line < LinesTotal() && !IsWhiteLine(line)) { // skip non-empty lines
                line++;
        }
        while (line < LinesTotal() && IsWhiteLine(line)) { // skip empty lines
                line++;
        }
        if (line < LinesTotal())
                return LineStart(line);
        else // end of a document
                return LineEnd(line-1);
}

CharClassify::cc Document::WordCharClass(unsigned char ch) {
        if ((SC_CP_UTF8 == dbcsCodePage) && (ch >= 0x80))
                return CharClassify::ccWord;
        return charClass.GetClass(ch);
}

/**
 * Used by commmands that want to select whole words.
 * Finds the start of word at pos when delta < 0 or the end of the word when delta >= 0.
 */
int Document::ExtendWordSelect(int pos, int delta, bool onlyWordCharacters) {
        CharClassify::cc ccStart = CharClassify::ccWord;
        if (delta < 0) {
                if (!onlyWordCharacters)
                        ccStart = WordCharClass(cb.CharAt(pos-1));
                while (pos > 0 && (WordCharClass(cb.CharAt(pos - 1)) == ccStart))
                        pos--;
        } else {
                if (!onlyWordCharacters && pos < Length())
                        ccStart = WordCharClass(cb.CharAt(pos));
                while (pos < (Length()) && (WordCharClass(cb.CharAt(pos)) == ccStart))
                        pos++;
        }
        return MovePositionOutsideChar(pos, delta, true);
}

/**
 * Find the start of the next word in either a forward (delta >= 0) or backwards direction
 * (delta < 0).
 * This is looking for a transition between character classes although there is also some
 * additional movement to transit white space.
 * Used by cursor movement by word commands.
 */
int Document::NextWordStart(int pos, int delta) {
        if (delta < 0) {
                while (pos > 0 && (WordCharClass(cb.CharAt(pos - 1)) == CharClassify::ccSpace))
                        pos--;
                if (pos > 0) {
                        CharClassify::cc ccStart = WordCharClass(cb.CharAt(pos-1));
                        while (pos > 0 && (WordCharClass(cb.CharAt(pos - 1)) == ccStart)) {
                                pos--;
                        }
                }
        } else {
                CharClassify::cc ccStart = WordCharClass(cb.CharAt(pos));
                while (pos < (Length()) && (WordCharClass(cb.CharAt(pos)) == ccStart))
                        pos++;
                while (pos < (Length()) && (WordCharClass(cb.CharAt(pos)) == CharClassify::ccSpace))
                        pos++;
        }
        return pos;
}

/**
 * Find the end of the next word in either a forward (delta >= 0) or backwards direction
 * (delta < 0).
 * This is looking for a transition between character classes although there is also some
 * additional movement to transit white space.
 * Used by cursor movement by word commands.
 */
int Document::NextWordEnd(int pos, int delta) {
        if (delta < 0) {
                if (pos > 0) {
                        CharClassify::cc ccStart = WordCharClass(cb.CharAt(pos-1));
                        if (ccStart != CharClassify::ccSpace) {
                                while (pos > 0 && WordCharClass(cb.CharAt(pos - 1)) == ccStart) {
                                        pos--;
                                }
                        }
                        while (pos > 0 && WordCharClass(cb.CharAt(pos - 1)) == CharClassify::ccSpace) {
                                pos--;
                        }
                }
        } else {
                while (pos < Length() && WordCharClass(cb.CharAt(pos)) == CharClassify::ccSpace) {
                        pos++;
                }
                if (pos < Length()) {
                        CharClassify::cc ccStart = WordCharClass(cb.CharAt(pos));
                        while (pos < Length() && WordCharClass(cb.CharAt(pos)) == ccStart) {
                                pos++;
                        }
                }
        }
        return pos;
}

/**
 * Check that the character at the given position is a word or punctuation character and that
 * the previous character is of a different character class.
 */
bool Document::IsWordStartAt(int pos) {
        if (pos > 0) {
                CharClassify::cc ccPos = WordCharClass(CharAt(pos));
                return (ccPos == CharClassify::ccWord || ccPos == CharClassify::ccPunctuation) &&
                        (ccPos != WordCharClass(CharAt(pos - 1)));
        }
        return true;
}

/**
 * Check that the character at the given position is a word or punctuation character and that
 * the next character is of a different character class.
 */
bool Document::IsWordEndAt(int pos) {
        if (pos < Length()) {
                CharClassify::cc ccPrev = WordCharClass(CharAt(pos-1));
                return (ccPrev == CharClassify::ccWord || ccPrev == CharClassify::ccPunctuation) &&
                        (ccPrev != WordCharClass(CharAt(pos)));
        }
        return true;
}

/**
 * Check that the given range is has transitions between character classes at both
 * ends and where the characters on the inside are word or punctuation characters.
 */
bool Document::IsWordAt(int start, int end) {
        return IsWordStartAt(start) && IsWordEndAt(end);
}

static inline char MakeLowerCase(char ch) {
        if (ch < 'A' || ch > 'Z')
                return ch;
        else
                return static_cast<char>(ch - 'A' + 'a');
}

static bool GoodTrailByte(int v) {
        return (v >= 0x80) && (v < 0xc0);
}

size_t Document::ExtractChar(int pos, char *bytes) {
        unsigned char ch = static_cast<unsigned char>(cb.CharAt(pos));
        size_t widthChar = UTF8CharLength(ch);
        bytes[0] = ch;
        for (size_t i=1; i<widthChar; i++) {
                bytes[i] = cb.CharAt(static_cast<int>(pos+i));
                if (!GoodTrailByte(static_cast<unsigned char>(bytes[i]))) { // Bad byte
                        widthChar = 1;
                }
        }
        return widthChar;
}

CaseFolderTable::CaseFolderTable() {
        for (size_t iChar=0; iChar<sizeof(mapping); iChar++) {
                mapping[iChar] = static_cast<char>(iChar);
        }
}

CaseFolderTable::~CaseFolderTable() {
}

size_t CaseFolderTable::Fold(char *folded, size_t sizeFolded, const char *mixed, size_t lenMixed) {
        if (lenMixed > sizeFolded) {
                return 0;
        } else {
                for (size_t i=0; i<lenMixed; i++) {
                        folded[i] = mapping[static_cast<unsigned char>(mixed[i])];
                }
                return lenMixed;
        }
}

void CaseFolderTable::SetTranslation(char ch, char chTranslation) {
        mapping[static_cast<unsigned char>(ch)] = chTranslation;
}

void CaseFolderTable::StandardASCII() {
        for (size_t iChar=0; iChar<sizeof(mapping); iChar++) {
                if (iChar >= 'A' && iChar <= 'Z') {
                        mapping[iChar] = static_cast<char>(iChar - 'A' + 'a');
                } else {
                        mapping[iChar] = static_cast<char>(iChar);
                }
        }
}

bool Document::MatchesWordOptions(bool word, bool wordStart, int pos, int length) {
        return (!word && !wordStart) ||
                        (word && IsWordAt(pos, pos + length)) ||
                        (wordStart && IsWordStartAt(pos));
}

/**
 * Find text in document, supporting both forward and backward
 * searches (just pass minPos > maxPos to do a backward search)
 * Has not been tested with backwards DBCS searches yet.
 */
long Document::FindText(int minPos, int maxPos, const char *search,
                        bool caseSensitive, bool word, bool wordStart, bool regExp, int flags,
                        int *length, CaseFolder *pcf) {
        if (*length <= 0)
                return minPos;
        if (regExp) {
                if (!regex)
                        regex = CreateRegexSearch(&charClass);
                return regex->FindText(this, minPos, maxPos, search, caseSensitive, word, wordStart, flags, length);
        } else {

                const bool forward = minPos <= maxPos;
                const int increment = forward ? 1 : -1;

                // Range endpoints should not be inside DBCS characters, but just in case, move them.
                const int startPos = MovePositionOutsideChar(minPos, increment, false);
                const int endPos = MovePositionOutsideChar(maxPos, increment, false);

                // Compute actual search ranges needed
                const int lengthFind = *length;

                //Platform::DebugPrintf("Find %d %d %s %d\n", startPos, endPos, ft->lpstrText, lengthFind);
                const int limitPos = Platform::Maximum(startPos, endPos);
                int pos = startPos;
                if (!forward) {
                        // Back all of a character
                        pos = NextPosition(pos, increment);
                }
                if (caseSensitive) {
                        const int endSearch = (startPos <= endPos) ? endPos - lengthFind + 1 : endPos;
                        while (forward ? (pos < endSearch) : (pos >= endSearch)) {
                                bool found = (pos + lengthFind) <= limitPos;
                                for (int indexSearch = 0; (indexSearch < lengthFind) && found; indexSearch++) {
                                        found = CharAt(pos + indexSearch) == search[indexSearch];
                                }
                                if (found && MatchesWordOptions(word, wordStart, pos, lengthFind)) {
                                        return pos;
                                }
                                if (!NextCharacter(pos, increment))
                                        break;
                        }
                } else if (SC_CP_UTF8 == dbcsCodePage) {
                        const size_t maxBytesCharacter = 4;
                        const size_t maxFoldingExpansion = 4;
                        std::vector<char> searchThing(lengthFind * maxBytesCharacter * maxFoldingExpansion + 1);
                        const int lenSearch = static_cast<int>(
                                pcf->Fold(&searchThing[0], searchThing.size(), search, lengthFind));
                        while (forward ? (pos < endPos) : (pos >= endPos)) {
                                int widthFirstCharacter = 0;
                                int indexDocument = 0;
                                int indexSearch = 0;
                                bool characterMatches = true;
                                while (characterMatches &&
                                        ((pos + indexDocument) < limitPos) &&
                                        (indexSearch < lenSearch)) {
                                        char bytes[maxBytesCharacter + 1];
                                        bytes[maxBytesCharacter] = 0;
                                        const int widthChar = static_cast<int>(ExtractChar(pos + indexDocument, bytes));
                                        if (!widthFirstCharacter)
                                                widthFirstCharacter = widthChar;
                                        if ((pos + indexDocument + widthChar) > limitPos)
                                                break;
                                        char folded[maxBytesCharacter * maxFoldingExpansion + 1];
                                        const int lenFlat = static_cast<int>(pcf->Fold(folded, sizeof(folded), bytes, widthChar));
                                        folded[lenFlat] = 0;
                                        // Does folded match the buffer
                                        characterMatches = 0 == memcmp(folded, &searchThing[0] + indexSearch, lenFlat);
                                        indexDocument += widthChar;
                                        indexSearch += lenFlat;
                                }
                                if (characterMatches && (indexSearch == static_cast<int>(lenSearch))) {
                                        if (MatchesWordOptions(word, wordStart, pos, indexDocument)) {
                                                *length = indexDocument;
                                                return pos;
                                        }
                                }
                                if (forward) {
                                        pos += widthFirstCharacter;
                                } else {
                                        if (!NextCharacter(pos, increment))
                                                break;
                                }
                        }
                } else if (dbcsCodePage) {
                        const size_t maxBytesCharacter = 2;
                        const size_t maxFoldingExpansion = 4;
                        std::vector<char> searchThing(lengthFind * maxBytesCharacter * maxFoldingExpansion + 1);
                        const int lenSearch = static_cast<int>(
                                pcf->Fold(&searchThing[0], searchThing.size(), search, lengthFind));
                        while (forward ? (pos < endPos) : (pos >= endPos)) {
                                int indexDocument = 0;
                                int indexSearch = 0;
                                bool characterMatches = true;
                                while (characterMatches &&
                                        ((pos + indexDocument) < limitPos) &&
                                        (indexSearch < lenSearch)) {
                                        char bytes[maxBytesCharacter + 1];
                                        bytes[0] = cb.CharAt(pos + indexDocument);
                                        const int widthChar = IsDBCSLeadByte(bytes[0]) ? 2 : 1;
                                        if (widthChar == 2)
                                                bytes[1] = cb.CharAt(pos + indexDocument + 1);
                                        if ((pos + indexDocument + widthChar) > limitPos)
                                                break;
                                        char folded[maxBytesCharacter * maxFoldingExpansion + 1];
                                        const int lenFlat = static_cast<int>(pcf->Fold(folded, sizeof(folded), bytes, widthChar));
                                        folded[lenFlat] = 0;
                                        // Does folded match the buffer
                                        characterMatches = 0 == memcmp(folded, &searchThing[0] + indexSearch, lenFlat);
                                        indexDocument += widthChar;
                                        indexSearch += lenFlat;
                                }
                                if (characterMatches && (indexSearch == static_cast<int>(lenSearch))) {
                                        if (MatchesWordOptions(word, wordStart, pos, indexDocument)) {
                                                *length = indexDocument;
                                                return pos;
                                        }
                                }
                                if (!NextCharacter(pos, increment))
                                        break;
                        }
                } else {
                        const int endSearch = (startPos <= endPos) ? endPos - lengthFind + 1 : endPos;
                        std::vector<char> searchThing(lengthFind + 1);
                        pcf->Fold(&searchThing[0], searchThing.size(), search, lengthFind);
                        while (forward ? (pos < endSearch) : (pos >= endSearch)) {
                                bool found = (pos + lengthFind) <= limitPos;
                                for (int indexSearch = 0; (indexSearch < lengthFind) && found; indexSearch++) {
                                        char ch = CharAt(pos + indexSearch);
                                        char folded[2];
                                        pcf->Fold(folded, sizeof(folded), &ch, 1);
                                        found = folded[0] == searchThing[indexSearch];
                                }
                                if (found && MatchesWordOptions(word, wordStart, pos, lengthFind)) {
                                        return pos;
                                }
                                if (!NextCharacter(pos, increment))
                                        break;
                        }
                }
        }
        //Platform::DebugPrintf("Not found\n");
        return -1;
}

const char *Document::SubstituteByPosition(const char *text, int *length) {
        if (regex)
                return regex->SubstituteByPosition(this, text, length);
        else
                return 0;
}

int Document::LinesTotal() const {
        return cb.Lines();
}

void Document::ChangeCase(Range r, bool makeUpperCase) {
        for (int pos = r.start; pos < r.end;) {
                int len = LenChar(pos);
                if (len == 1) {
                        char ch = CharAt(pos);
                        if (makeUpperCase) {
                                if (IsLowerCase(ch)) {
                                        ChangeChar(pos, static_cast<char>(MakeUpperCase(ch)));
                                }
                        } else {
                                if (IsUpperCase(ch)) {
                                        ChangeChar(pos, static_cast<char>(MakeLowerCase(ch)));
                                }
                        }
                }
                pos += len;
        }
}

void Document::SetDefaultCharClasses(bool includeWordClass) {
    charClass.SetDefaultCharClasses(includeWordClass);
}

void Document::SetCharClasses(const unsigned char *chars, CharClassify::cc newCharClass) {
    charClass.SetCharClasses(chars, newCharClass);
}

void Document::SetStylingBits(int bits) {
        stylingBits = bits;
        stylingBitsMask = (1 << stylingBits) - 1;
}

void SCI_METHOD Document::StartStyling(int position, char mask) {
        stylingMask = mask;
        endStyled = position;
}

bool SCI_METHOD Document::SetStyleFor(int length, char style) {
        if (enteredStyling != 0) {
                return false;
        } else {
                enteredStyling++;
                style &= stylingMask;
                int prevEndStyled = endStyled;
                if (cb.SetStyleFor(endStyled, length, style, stylingMask)) {
                        DocModification mh(SC_MOD_CHANGESTYLE | SC_PERFORMED_USER,
                                           prevEndStyled, length);
                        NotifyModified(mh);
                }
                endStyled += length;
                enteredStyling--;
                return true;
        }
}

bool SCI_METHOD Document::SetStyles(int length, const char *styles) {
        if (enteredStyling != 0) {
                return false;
        } else {
                enteredStyling++;
                bool didChange = false;
                int startMod = 0;
                int endMod = 0;
                for (int iPos = 0; iPos < length; iPos++, endStyled++) {
                        PLATFORM_ASSERT(endStyled < Length());
                        if (cb.SetStyleAt(endStyled, styles[iPos], stylingMask)) {
                                if (!didChange) {
                                        startMod = endStyled;
                                }
                                didChange = true;
                                endMod = endStyled;
                        }
                }
                if (didChange) {
                        DocModification mh(SC_MOD_CHANGESTYLE | SC_PERFORMED_USER,
                                           startMod, endMod - startMod + 1);
                        NotifyModified(mh);
                }
                enteredStyling--;
                return true;
        }
}

void Document::EnsureStyledTo(int pos) {
        if ((enteredStyling == 0) && (pos > GetEndStyled())) {
                IncrementStyleClock();
                if (pli && !pli->UseContainerLexing()) {
                        int lineEndStyled = LineFromPosition(GetEndStyled());
                        int endStyledTo = LineStart(lineEndStyled);
                        pli->Colourise(endStyledTo, pos);
                } else {
                        // Ask the watchers to style, and stop as soon as one responds.
                        for (int i = 0; pos > GetEndStyled() && i < lenWatchers; i++) {
                                watchers[i].watcher->NotifyStyleNeeded(this, watchers[i].userData, pos);
                        }
                }
        }
}

void Document::LexerChanged() {
        // Tell the watchers the lexer has changed.
        for (int i = 0; i < lenWatchers; i++) {
                watchers[i].watcher->NotifyLexerChanged(this, watchers[i].userData);
        }
}

int SCI_METHOD Document::SetLineState(int line, int state) {
        int statePrevious = static_cast<LineState *>(perLineData[ldState])->SetLineState(line, state);
        if (state != statePrevious) {
                DocModification mh(SC_MOD_CHANGELINESTATE, LineStart(line), 0, 0, 0, line);
                NotifyModified(mh);
        }
        return statePrevious;
}

int SCI_METHOD Document::GetLineState(int line) const {
        return static_cast<LineState *>(perLineData[ldState])->GetLineState(line);
}

int Document::GetMaxLineState() {
        return static_cast<LineState *>(perLineData[ldState])->GetMaxLineState();
}

void SCI_METHOD Document::ChangeLexerState(int start, int end) {
        DocModification mh(SC_MOD_LEXERSTATE, start, end-start, 0, 0, 0);
        NotifyModified(mh);
}

StyledText Document::MarginStyledText(int line) {
        LineAnnotation *pla = static_cast<LineAnnotation *>(perLineData[ldMargin]);
        return StyledText(pla->Length(line), pla->Text(line),
                pla->MultipleStyles(line), pla->Style(line), pla->Styles(line));
}

void Document::MarginSetText(int line, const char *text) {
        static_cast<LineAnnotation *>(perLineData[ldMargin])->SetText(line, text);
        DocModification mh(SC_MOD_CHANGEMARGIN, LineStart(line), 0, 0, 0, line);
        NotifyModified(mh);
}

void Document::MarginSetStyle(int line, int style) {
        static_cast<LineAnnotation *>(perLineData[ldMargin])->SetStyle(line, style);
}

void Document::MarginSetStyles(int line, const unsigned char *styles) {
        static_cast<LineAnnotation *>(perLineData[ldMargin])->SetStyles(line, styles);
}

int Document::MarginLength(int line) const {
        return static_cast<LineAnnotation *>(perLineData[ldMargin])->Length(line);
}

void Document::MarginClearAll() {
        int maxEditorLine = LinesTotal();
        for (int l=0; l<maxEditorLine; l++)
                MarginSetText(l, 0);
        // Free remaining data
        static_cast<LineAnnotation *>(perLineData[ldMargin])->ClearAll();
}

bool Document::AnnotationAny() const {
        return static_cast<LineAnnotation *>(perLineData[ldAnnotation])->AnySet();
}

StyledText Document::AnnotationStyledText(int line) {
        LineAnnotation *pla = static_cast<LineAnnotation *>(perLineData[ldAnnotation]);
        return StyledText(pla->Length(line), pla->Text(line),
                pla->MultipleStyles(line), pla->Style(line), pla->Styles(line));
}

void Document::AnnotationSetText(int line, const char *text) {
        const int linesBefore = AnnotationLines(line);
        static_cast<LineAnnotation *>(perLineData[ldAnnotation])->SetText(line, text);
        const int linesAfter = AnnotationLines(line);
        DocModification mh(SC_MOD_CHANGEANNOTATION, LineStart(line), 0, 0, 0, line);
        mh.annotationLinesAdded = linesAfter - linesBefore;
        NotifyModified(mh);
}

void Document::AnnotationSetStyle(int line, int style) {
        static_cast<LineAnnotation *>(perLineData[ldAnnotation])->SetStyle(line, style);
        DocModification mh(SC_MOD_CHANGEANNOTATION, LineStart(line), 0, 0, 0, line);
        NotifyModified(mh);
}

void Document::AnnotationSetStyles(int line, const unsigned char *styles) {
        static_cast<LineAnnotation *>(perLineData[ldAnnotation])->SetStyles(line, styles);
}

int Document::AnnotationLength(int line) const {
        return static_cast<LineAnnotation *>(perLineData[ldAnnotation])->Length(line);
}

int Document::AnnotationLines(int line) const {
        return static_cast<LineAnnotation *>(perLineData[ldAnnotation])->Lines(line);
}

void Document::AnnotationClearAll() {
        int maxEditorLine = LinesTotal();
        for (int l=0; l<maxEditorLine; l++)
                AnnotationSetText(l, 0);
        // Free remaining data
        static_cast<LineAnnotation *>(perLineData[ldAnnotation])->ClearAll();
}

void Document::IncrementStyleClock() {
        styleClock = (styleClock + 1) % 0x100000;
}

void SCI_METHOD Document::DecorationFillRange(int position, int value, int fillLength) {
        if (decorations.FillRange(position, value, fillLength)) {
                DocModification mh(SC_MOD_CHANGEINDICATOR | SC_PERFORMED_USER,
                                                        position, fillLength);
                NotifyModified(mh);
        }
}

bool Document::AddWatcher(DocWatcher *watcher, void *userData) {
        for (int i = 0; i < lenWatchers; i++) {
                if ((watchers[i].watcher == watcher) &&
                        (watchers[i].userData == userData))
                        return false;
        }
        WatcherWithUserData *pwNew = new WatcherWithUserData[lenWatchers + 1];
        for (int j = 0; j < lenWatchers; j++)
                pwNew[j] = watchers[j];
        pwNew[lenWatchers].watcher = watcher;
        pwNew[lenWatchers].userData = userData;
        delete []watchers;
        watchers = pwNew;
        lenWatchers++;
        return true;
}

bool Document::RemoveWatcher(DocWatcher *watcher, void *userData) {
        for (int i = 0; i < lenWatchers; i++) {
                if ((watchers[i].watcher == watcher) &&
                        (watchers[i].userData == userData)) {
                        if (lenWatchers == 1) {
                                delete []watchers;
                                watchers = 0;
                                lenWatchers = 0;
                        } else {
                                WatcherWithUserData *pwNew = new WatcherWithUserData[lenWatchers];
                                for (int j = 0; j < lenWatchers - 1; j++) {
                                        pwNew[j] = (j < i) ? watchers[j] : watchers[j + 1];
                                }
                                delete []watchers;
                                watchers = pwNew;
                                lenWatchers--;
                        }
                        return true;
                }
        }
        return false;
}

void Document::NotifyModifyAttempt() {
        for (int i = 0; i < lenWatchers; i++) {
                watchers[i].watcher->NotifyModifyAttempt(this, watchers[i].userData);
        }
}

void Document::NotifySavePoint(bool atSavePoint) {
        for (int i = 0; i < lenWatchers; i++) {
                watchers[i].watcher->NotifySavePoint(this, watchers[i].userData, atSavePoint);
        }
}

void Document::NotifyModified(DocModification mh) {
        if (mh.modificationType & SC_MOD_INSERTTEXT) {
                decorations.InsertSpace(mh.position, mh.length);
        } else if (mh.modificationType & SC_MOD_DELETETEXT) {
                decorations.DeleteRange(mh.position, mh.length);
        }
        for (int i = 0; i < lenWatchers; i++) {
                watchers[i].watcher->NotifyModified(this, mh, watchers[i].userData);
        }
}

bool Document::IsWordPartSeparator(char ch) {
        return (WordCharClass(ch) == CharClassify::ccWord) && IsPunctuation(ch);
}

int Document::WordPartLeft(int pos) {
        if (pos > 0) {
                --pos;
                char startChar = cb.CharAt(pos);
                if (IsWordPartSeparator(startChar)) {
                        while (pos > 0 && IsWordPartSeparator(cb.CharAt(pos))) {
                                --pos;
                        }
                }
                if (pos > 0) {
                        startChar = cb.CharAt(pos);
                        --pos;
                        if (IsLowerCase(startChar)) {
                                while (pos > 0 && IsLowerCase(cb.CharAt(pos)))
                                        --pos;
                                if (!IsUpperCase(cb.CharAt(pos)) && !IsLowerCase(cb.CharAt(pos)))
                                        ++pos;
                        } else if (IsUpperCase(startChar)) {
                                while (pos > 0 && IsUpperCase(cb.CharAt(pos)))
                                        --pos;
                                if (!IsUpperCase(cb.CharAt(pos)))
                                        ++pos;
                        } else if (IsADigit(startChar)) {
                                while (pos > 0 && IsADigit(cb.CharAt(pos)))
                                        --pos;
                                if (!IsADigit(cb.CharAt(pos)))
                                        ++pos;
                        } else if (IsPunctuation(startChar)) {
                                while (pos > 0 && IsPunctuation(cb.CharAt(pos)))
                                        --pos;
                                if (!IsPunctuation(cb.CharAt(pos)))
                                        ++pos;
                        } else if (isspacechar(startChar)) {
                                while (pos > 0 && isspacechar(cb.CharAt(pos)))
                                        --pos;
                                if (!isspacechar(cb.CharAt(pos)))
                                        ++pos;
                        } else if (!isascii(startChar)) {
                                while (pos > 0 && !isascii(cb.CharAt(pos)))
                                        --pos;
                                if (isascii(cb.CharAt(pos)))
                                        ++pos;
                        } else {
                                ++pos;
                        }
                }
        }
        return pos;
}

int Document::WordPartRight(int pos) {
        char startChar = cb.CharAt(pos);
        int length = Length();
        if (IsWordPartSeparator(startChar)) {
                while (pos < length && IsWordPartSeparator(cb.CharAt(pos)))
                        ++pos;
                startChar = cb.CharAt(pos);
        }
        if (!isascii(startChar)) {
                while (pos < length && !isascii(cb.CharAt(pos)))
                        ++pos;
        } else if (IsLowerCase(startChar)) {
                while (pos < length && IsLowerCase(cb.CharAt(pos)))
                        ++pos;
        } else if (IsUpperCase(startChar)) {
                if (IsLowerCase(cb.CharAt(pos + 1))) {
                        ++pos;
                        while (pos < length && IsLowerCase(cb.CharAt(pos)))
                                ++pos;
                } else {
                        while (pos < length && IsUpperCase(cb.CharAt(pos)))
                                ++pos;
                }
                if (IsLowerCase(cb.CharAt(pos)) && IsUpperCase(cb.CharAt(pos - 1)))
                        --pos;
        } else if (IsADigit(startChar)) {
                while (pos < length && IsADigit(cb.CharAt(pos)))
                        ++pos;
        } else if (IsPunctuation(startChar)) {
                while (pos < length && IsPunctuation(cb.CharAt(pos)))
                        ++pos;
        } else if (isspacechar(startChar)) {
                while (pos < length && isspacechar(cb.CharAt(pos)))
                        ++pos;
        } else {
                ++pos;
        }
        return pos;
}

bool IsLineEndChar(char c) {
        return (c == '\n' || c == '\r');
}

int Document::ExtendStyleRange(int pos, int delta, bool singleLine) {
        int sStart = cb.StyleAt(pos);
        if (delta < 0) {
                while (pos > 0 && (cb.StyleAt(pos) == sStart) && (!singleLine || !IsLineEndChar(cb.CharAt(pos))))
                        pos--;
                pos++;
        } else {
                while (pos < (Length()) && (cb.StyleAt(pos) == sStart) && (!singleLine || !IsLineEndChar(cb.CharAt(pos))))
                        pos++;
        }
        return pos;
}

static char BraceOpposite(char ch) {
        switch (ch) {
        case '(':
                return ')';
        case ')':
                return '(';
        case '[':
                return ']';
        case ']':
                return '[';
        case '{':
                return '}';
        case '}':
                return '{';
        case '<':
                return '>';
        case '>':
                return '<';
        default:
                return '\0';
        }
}

// TODO: should be able to extend styled region to find matching brace
int Document::BraceMatch(int position, int /*maxReStyle*/) {
        char chBrace = CharAt(position);
        char chSeek = BraceOpposite(chBrace);
        if (chSeek == '\0')
                return - 1;
        char styBrace = static_cast<char>(StyleAt(position) & stylingBitsMask);
        int direction = -1;
        if (chBrace == '(' || chBrace == '[' || chBrace == '{' || chBrace == '<')
                direction = 1;
        int depth = 1;
        position = NextPosition(position, direction);
        while ((position >= 0) && (position < Length())) {
                char chAtPos = CharAt(position);
                char styAtPos = static_cast<char>(StyleAt(position) & stylingBitsMask);
                if ((position > GetEndStyled()) || (styAtPos == styBrace)) {
                        if (chAtPos == chBrace)
                                depth++;
                        if (chAtPos == chSeek)
                                depth--;
                        if (depth == 0)
                                return position;
                }
                int positionBeforeMove = position;
                position = NextPosition(position, direction);
                if (position == positionBeforeMove)
                        break;
        }
        return - 1;
}

/**
 * Implementation of RegexSearchBase for the default built-in regular expression engine
 */
class BuiltinRegex : public RegexSearchBase {
public:
        BuiltinRegex(CharClassify *charClassTable) : search(charClassTable), substituted(NULL) {}

        virtual ~BuiltinRegex() {
                delete substituted;
        }

        virtual long FindText(Document *doc, int minPos, int maxPos, const char *s,
                        bool caseSensitive, bool word, bool wordStart, int flags,
                        int *length);

        virtual const char *SubstituteByPosition(Document *doc, const char *text, int *length);

private:
        RESearch search;
        char *substituted;
};

// Define a way for the Regular Expression code to access the document
class DocumentIndexer : public CharacterIndexer {
        Document *pdoc;
        int end;
public:
        DocumentIndexer(Document *pdoc_, int end_) :
                pdoc(pdoc_), end(end_) {
        }

        virtual ~DocumentIndexer() {
        }

        virtual char CharAt(int index) {
                if (index < 0 || index >= end)
                        return 0;
                else
                        return pdoc->CharAt(index);
        }
};

long BuiltinRegex::FindText(Document *doc, int minPos, int maxPos, const char *s,
                        bool caseSensitive, bool word, bool, int flags,
                        int *length) {
        bool posix = (flags & SCFIND_POSIX) != 0;
        int increment = (minPos <= maxPos) ? 1 : -1;

        int startPos = minPos;
        int endPos = maxPos;

        // Range endpoints should not be inside DBCS characters, but just in case, move them.
        startPos = doc->MovePositionOutsideChar(startPos, 1, false);
        endPos = doc->MovePositionOutsideChar(endPos, 1, false);

        const char *errmsg = search.Compile(s, *length, caseSensitive, posix);
        if (errmsg) {
                return -1;
        }
        // Find a variable in a property file: \$(\([A-Za-z0-9_.]+\))
        // Replace first '.' with '-' in each property file variable reference:
        //     Search: \$(\([A-Za-z0-9_-]+\)\.\([A-Za-z0-9_.]+\))
        //     Replace: $(\1-\2)
        int lineRangeStart = doc->LineFromPosition(startPos);
        int lineRangeEnd = doc->LineFromPosition(endPos);
        if ((increment == 1) &&
                (startPos >= doc->LineEnd(lineRangeStart)) &&
                (lineRangeStart < lineRangeEnd)) {
                // the start position is at end of line or between line end characters.
                lineRangeStart++;
                startPos = doc->LineStart(lineRangeStart);
        } else if ((increment == -1) &&
                   (startPos <= doc->LineStart(lineRangeStart)) &&
                   (lineRangeStart > lineRangeEnd)) {
                // the start position is at beginning of line.
                lineRangeStart--;
                startPos = doc->LineEnd(lineRangeStart);
        }
        int pos = -1;
        int lenRet = 0;
        char searchEnd = s[*length - 1];
        char searchEndPrev = (*length > 1) ? s[*length - 2] : '\0';
        int lineRangeBreak = lineRangeEnd + increment;
        for (int line = lineRangeStart; line != lineRangeBreak; line += increment) {
                int startOfLine = doc->LineStart(line);
                int endOfLine = doc->LineEnd(line);
                if (increment == 1) {
                        if (line == lineRangeStart) {
                                if ((startPos != startOfLine) && (s[0] == '^'))
                                        continue;       // Can't match start of line if start position after start of line
                                startOfLine = startPos;
                        }
                        if (line == lineRangeEnd) {
                                if ((endPos != endOfLine) && (searchEnd == '$') && (searchEndPrev != '\\'))
                                        continue;       // Can't match end of line if end position before end of line
                                endOfLine = endPos;
                        }
                } else {
                        if (line == lineRangeEnd) {
                                if ((endPos != startOfLine) && (s[0] == '^'))
                                        continue;       // Can't match start of line if end position after start of line
                                startOfLine = endPos;
                        }
                        if (line == lineRangeStart) {
                                if ((startPos != endOfLine) && (searchEnd == '$') && (searchEndPrev != '\\'))
                                        continue;       // Can't match end of line if start position before end of line
                                endOfLine = startPos;
                        }
                }

                DocumentIndexer di(doc, endOfLine);
                int success = search.Execute(di, startOfLine, endOfLine, word);//[mjr - added 'word']
                if (success) {
                        pos = search.bopat[0];
                        lenRet = search.eopat[0] - search.bopat[0];
                        // There can be only one start of a line, so no need to look for last match in line
                        if ((increment == -1) && (s[0] != '^')) {
                                // Check for the last match on this line.
                                int repetitions = 1000; // Break out of infinite loop
                                while (success && (search.eopat[0] <= endOfLine) && (repetitions--)) {
                                        success = search.Execute(di, pos+1, endOfLine, word);//[mjr - added 'word']
                                        if (success) {
                                                if (search.eopat[0] <= minPos) {
                                                        pos = search.bopat[0];
                                                        lenRet = search.eopat[0] - search.bopat[0];
                                                } else {
                                                        success = 0;
                                                }
                                        }
                                }
                        }
                        break;
                }
        }
        *length = lenRet;
        return pos;
}

const char *BuiltinRegex::SubstituteByPosition(Document *doc, const char *text, int *length) {
        delete []substituted;
        substituted = 0;
        DocumentIndexer di(doc, doc->Length());
        if (!search.GrabMatches(di))
                return 0;
        unsigned int lenResult = 0;
        for (int i = 0; i < *length; i++) {
                if (text[i] == '\\') {
                        if (text[i + 1] >= '1' && text[i + 1] <= '9') {
                                unsigned int patNum = text[i + 1] - '0';
                                lenResult += search.eopat[patNum] - search.bopat[patNum];
                                i++;
                        } else {
                                switch (text[i + 1]) {
                                case 'a':
                                case 'b':
                                case 'f':
                                case 'n':
                                case 'r':
                                case 't':
                                case 'v':
                                case '\\':
                                        i++;
                                }
                                lenResult++;
                        }
                } else {
                        lenResult++;
                }
        }
        substituted = new char[lenResult + 1];
        char *o = substituted;
        for (int j = 0; j < *length; j++) {
                if (text[j] == '\\') {
                        if (text[j + 1] >= '1' && text[j + 1] <= '9') {
                                unsigned int patNum = text[j + 1] - '0';
                                unsigned int len = search.eopat[patNum] - search.bopat[patNum];
                                if (search.pat[patNum]) // Will be null if try for a match that did not occur
                                        memcpy(o, search.pat[patNum], len);
                                o += len;
                                j++;
                        } else {
                                j++;
                                switch (text[j]) {
                                case 'a':
                                        *o++ = '\a';
                                        break;
                                case 'b':
                                        *o++ = '\b';
                                        break;
                                case 'f':
                                        *o++ = '\f';
                                        break;
                                case 'n':
                                        *o++ = '\n';
                                        break;
                                case 'r':
                                        *o++ = '\r';
                                        break;
                                case 't':
                                        *o++ = '\t';
                                        break;
                                case 'v':
                                        *o++ = '\v';
                                        break;
                                case '\\':
                                        *o++ = '\\';
                                        break;
                                default:
                                        *o++ = '\\';
                                        j--;
                                }
                        }
                } else {
                        *o++ = text[j];
                }
        }
        *o = '\0';
        *length = lenResult;
        return substituted;
}

#ifndef SCI_OWNREGEX

#ifdef SCI_NAMESPACE

RegexSearchBase *Scintilla::CreateRegexSearch(CharClassify *charClassTable) {
        return new BuiltinRegex(charClassTable);
}

#else

RegexSearchBase *CreateRegexSearch(CharClassify *charClassTable) {
        return new BuiltinRegex(charClassTable);
}

#endif

#endif
