#include <index_consistency.h>

Public Member Functions
	IndexConsistency (OperationContext opCtx, Collection collection, NamespaceString nss, RecordStore *recordStore, std::unique_ptr< Lock::CollectionLock > collLk, const bool background)

void	addDocKey (const KeyString &ks, int indexNumber)
	Helper functions for `_addDocKey`, `_removeDocKey`, `_addIndexKey`, and `_removeIndexKey` for concurrency control. More...

void	removeDocKey (const KeyString &ks, int indexNumber)

void	addIndexKey (const KeyString &ks, int indexNumber)

void	removeIndexKey (const KeyString &ks, int indexNumber)

void	addLongIndexKey (int indexNumber)
	Add one to the `_longKeys` count for the given `indexNs`. More...

int64_t	getNumKeys (int indexNumber) const
	Returns the number of index entries for the given `indexNs`. More...

int64_t	getNumLongKeys (int indexNumber) const
	Returns the number of long keys that were not indexed for the given `indexNs`. More...

int64_t	getNumRecords (int indexNumber) const
	Return the number of records with keys for the given `indexNs`. More...

bool	haveEntryMismatch () const
	Returns true if any value in the `_indexKeyCount` map is not equal to 0, otherwise return false. More...

int64_t	getNumExtraIndexKeys (int indexNumber) const
	Index entries may be added or removed by concurrent writes during the index scan phase, after establishing the point of validity. More...

void	applyChange (const IndexDescriptor *descriptor, const boost::optional< IndexKeyEntry > &indexEntry, ValidationOperation operation)
	This is the entry point for the IndexObserver to apply its observed changes while it is listening for changes in the IndexAccessMethod. More...

void	nextStage ()
	Moves the `_stage` variable to the next corresponding stage in the following order: `DOCUMENT` -> `INDEX` `INDEX` -> `NONE` `NONE` -> `NONE` More...

ValidationStage	getStage () const
	Returns the `_stage` that the validation is on. More...

void	setLastProcessedRecordId (RecordId recordId)
	Sets `_lastProcessedRecordId` to `recordId`. More...

void	setLastProcessedIndexEntry (const IndexDescriptor &descriptor, const boost::optional< IndexKeyEntry > &indexEntry)
	Sets `_lastProcessedIndexEntry` to the KeyString of `indexEntry`. More...

void	notifyStartIndex (int indexNumber)
	Informs the IndexConsistency instance that the index scan is beginning to scan the index with namespace `indexNs`. More...

void	notifyDoneIndex (int indexNumber)
	Informs the IndexConsistency instance that the index scan has finished scanning the index with namespace `indexNs`. More...

int	getIndexNumber (const std::string &indexNs)
	Returns the index number for the corresponding index namespace's. More...

bool	shouldGetNewSnapshot (const RecordId recordId) const
	Returns true if a new snapshot should be accquired. More...

bool	shouldGetNewSnapshot (const KeyString &keyString) const
	Returns true if a new snapshot should be accquired. More...

void	relockCollectionWithMode (LockMode mode)
	Gives up the lock that the collection is currently held in and requests the the collection again in LockMode `mode` More...

bool	scanLimitHit ()
	Returns true if the ElapsedTracker says its time to yield during background validation. More...

Private Member Functions
void	_addDocKey_inlock (const KeyString &ks, int indexNumber)
	Given the document's key KeyString, increment the corresponding `_indexKeyCount` by hashing it. More...

void	_removeDocKey_inlock (const KeyString &ks, int indexNumber)
	Given the document's key KeyString, decrement the corresponding `_indexKeyCount` by hashing it. More...

void	_addIndexKey_inlock (const KeyString &ks, int indexNumber)
	Given the index entry's KeyString, decrement the corresponding `_indexKeyCount` by hashing it. More...

void	_removeIndexKey_inlock (const KeyString &ks, int indexNumber)
	Given the index entry's KeyString, increment the corresponding `_indexKeyCount` by hashing it. More...

bool	_isIndexFinished_inlock (int indexNumber) const
	Returns true if the index for the given `indexNs` has finished being scanned by the validation, otherwise it returns false. More...

bool	_isIndexScanning_inlock (int indexNumber) const
	Returns true if this is the current `indexNs` being scanned by validation, otherwise it returns false. More...

void	_setYieldAtRecord_inlock (const RecordId recordId)
	Allows the IndexObserver to set a yield point at `recordId` so that during the collection scan we must yield before processing the record. More...

void	_setYieldAtIndexEntry_inlock (const KeyString &keyString)
	Allows the IndexObserver to set a yield point at the KeyString of `indexEntry` so that during the index scan we must yield before processing the index entry. More...

bool	_isBeforeLastProcessedRecordId_inlock (RecordId recordId) const
	Returns true if the `recordId` is before or equal to the last processed RecordId. More...

bool	_isBeforeLastProcessedIndexEntry_inlock (const KeyString &keyString) const
	Returns true if the `keyString` is before or equal to the last processed index entry. More...

uint32_t	_hashKeyString (const KeyString &ks, int indexNumbers) const
	Returns a hashed value from the given KeyString and index namespace. More...

Status	_throwExceptionIfError ()
	Used alongside `yield()` and `relockCollectionWithMode()` to ensure that after the execution of them it is safe to continue validating. More...

Private Attributes
OperationContext *	_opCtx

Collection *	_collection

const NamespaceString	_nss

const RecordStore *	_recordStore

std::unique_ptr< Lock::CollectionLock >	_collLk

const bool	_isBackground

ElapsedTracker	_tracker

std::map< uint32_t, uint32_t >	_indexKeyCount

std::map< std::string, int >	_indexNumber

std::map< int, IndexInfo >	_indexesInfo

boost::optional< RecordId >	_lastProcessedRecordId = boost::none

std::unique_ptr< KeyString >	_lastProcessedIndexEntry = nullptr

int	_currentIndex = -1

ValidationStage	_stage = ValidationStage::DOCUMENT

boost::optional< RecordId >	_yieldAtRecordId = boost::none

std::unique_ptr< KeyString >	_yieldAtIndexEntry = nullptr

int	_numErrorsRecorded = 0

stdx::mutex	_classMutex

Static Private Attributes
static const int	_kErrorThreshold = 100

Constructor & Destructor Documentation

◆ IndexConsistency()

mongo::IndexConsistency::IndexConsistency	(	OperationContext *	opCtx,
		Collection *	collection,
		NamespaceString	nss,
		RecordStore *	recordStore,
		std::unique_ptr< Lock::CollectionLock >	collLk,
		const bool	background
	)

Member Function Documentation

◆ _addDocKey_inlock()

void mongo::IndexConsistency::_addDocKey_inlock	(	const KeyString &	ks,
		int	indexNumber
	)

private

Given the document's key KeyString, increment the corresponding _indexKeyCount by hashing it.

◆ _addIndexKey_inlock()

void mongo::IndexConsistency::_addIndexKey_inlock	(	const KeyString &	ks,
		int	indexNumber
	)

private

Given the index entry's KeyString, decrement the corresponding _indexKeyCount by hashing it.

◆ _hashKeyString()

uint32_t mongo::IndexConsistency::_hashKeyString	(	const KeyString &	ks,
		int	indexNumbers
	)		const

private

Returns a hashed value from the given KeyString and index namespace.

◆ _isBeforeLastProcessedIndexEntry_inlock()

bool mongo::IndexConsistency::_isBeforeLastProcessedIndexEntry_inlock ( const KeyString & keyString ) const

private

Returns true if the keyString is before or equal to the last processed index entry.

◆ _isBeforeLastProcessedRecordId_inlock()

bool mongo::IndexConsistency::_isBeforeLastProcessedRecordId_inlock ( RecordId recordId ) const

private

Returns true if the recordId is before or equal to the last processed RecordId.

◆ _isIndexFinished_inlock()

bool mongo::IndexConsistency::_isIndexFinished_inlock ( int indexNumber ) const

private

Returns true if the index for the given indexNs has finished being scanned by the validation, otherwise it returns false.

◆ _isIndexScanning_inlock()

bool mongo::IndexConsistency::_isIndexScanning_inlock ( int indexNumber ) const

private

Returns true if this is the current indexNs being scanned by validation, otherwise it returns false.

◆ _removeDocKey_inlock()

void mongo::IndexConsistency::_removeDocKey_inlock	(	const KeyString &	ks,
		int	indexNumber
	)

private

Given the document's key KeyString, decrement the corresponding _indexKeyCount by hashing it.

◆ _removeIndexKey_inlock()

void mongo::IndexConsistency::_removeIndexKey_inlock	(	const KeyString &	ks,
		int	indexNumber
	)

private

Given the index entry's KeyString, increment the corresponding _indexKeyCount by hashing it.

◆ _setYieldAtIndexEntry_inlock()

void mongo::IndexConsistency::_setYieldAtIndexEntry_inlock ( const KeyString & keyString )

private

Allows the IndexObserver to set a yield point at the KeyString of indexEntry so that during the index scan we must yield before processing the index entry.

This is a preventive measure so the index scan doesn't scan stale index entries.

◆ _setYieldAtRecord_inlock()

void mongo::IndexConsistency::_setYieldAtRecord_inlock ( const RecordId recordId )

private

Allows the IndexObserver to set a yield point at recordId so that during the collection scan we must yield before processing the record.

This is a preventive measure so the collection scan doesn't scan stale records.

◆ _throwExceptionIfError()

Status mongo::IndexConsistency::_throwExceptionIfError ( )

private

Used alongside yield() and relockCollectionWithMode() to ensure that after the execution of them it is safe to continue validating.

Validation can be stopped for a number of reasons including: 1) The database was dropped. 2) The collection was dropped. 3) An index was added or removed in the collection being validated. 4) The operation was killed.

◆ addDocKey()

void mongo::IndexConsistency::addDocKey	(	const KeyString &	ks,
		int	indexNumber
	)

Helper functions for _addDocKey, _removeDocKey, _addIndexKey, and _removeIndexKey for concurrency control.

◆ addIndexKey()

void mongo::IndexConsistency::addIndexKey	(	const KeyString &	ks,
		int	indexNumber
	)

◆ addLongIndexKey()

void mongo::IndexConsistency::addLongIndexKey ( int indexNumber )

Add one to the _longKeys count for the given indexNs.

This is required because index keys > KeyString::kMaxKeyBytes are not indexed.

◆ applyChange()

void mongo::IndexConsistency::applyChange	(	const IndexDescriptor *	descriptor,
		const boost::optional< IndexKeyEntry > &	indexEntry,
		ValidationOperation	operation
	)

This is the entry point for the IndexObserver to apply its observed changes while it is listening for changes in the IndexAccessMethod.

This method ensures that during the collection scan stage, inserted, removed and updated documents are reflected in the index key counts. It does this by: 1) Setting the yield point for the collection scan to inform us when we should get a new snapshot so we won't scan stale records. 2) Calling the appropriate addDocKey and removeDocKey functions if the record comes before or equal to our last processed RecordId.

The IndexObserver will call this method while it is observing changes during the index scan stage of the collection validation. It ensures we maintain a pre-image of the indexes since we established the point of validity, which was determined when the collection scan stage completed. It does this by: 1) Setting the yield point for the index scan to inform us when we should get a new snapshot so we won't scan stale index entries. The difference between this and the collection scan is that it will only set the yield point for the index that is currently being scanned, since when we start the next index, we will yield before we begin and we would have the latest snapshot. 2) Calling the appropriate addIndexKey and removeIndexKey functions for indexes that haven't started scanning and are not finished, or they are scanning the index and the index changes are after the last processed index entry. 3) In addition, we maintain the number of external index changes here so that after we finish the index scan, we can remove the extra number of operations that happened after the point of validity.

◆ getIndexNumber()

int mongo::IndexConsistency::getIndexNumber ( const std::string & indexNs )

Returns the index number for the corresponding index namespace's.

◆ getNumExtraIndexKeys()

int64_t mongo::IndexConsistency::getNumExtraIndexKeys ( int indexNumber ) const

Index entries may be added or removed by concurrent writes during the index scan phase, after establishing the point of validity.

We need to account for these additions and removals so that when we validate the index key count, we also have a pre-image of the index counts and won't get incorrect results because of the extra index entries we may or may not have scanned.

◆ getNumKeys()

int64_t mongo::IndexConsistency::getNumKeys ( int indexNumber ) const

Returns the number of index entries for the given indexNs.

◆ getNumLongKeys()

int64_t mongo::IndexConsistency::getNumLongKeys ( int indexNumber ) const

Returns the number of long keys that were not indexed for the given indexNs.

◆ getNumRecords()

int64_t mongo::IndexConsistency::getNumRecords ( int indexNumber ) const

Return the number of records with keys for the given indexNs.

◆ getStage()

ValidationStage mongo::IndexConsistency::getStage ( ) const

Returns the _stage that the validation is on.

◆ haveEntryMismatch()

bool mongo::IndexConsistency::haveEntryMismatch ( ) const

Returns true if any value in the _indexKeyCount map is not equal to 0, otherwise return false.

◆ nextStage()

void mongo::IndexConsistency::nextStage ( )

Moves the _stage variable to the next corresponding stage in the following order: DOCUMENT -> INDEX INDEX -> NONE NONE -> NONE

◆ notifyDoneIndex()

void mongo::IndexConsistency::notifyDoneIndex ( int indexNumber )

Informs the IndexConsistency instance that the index scan has finished scanning the index with namespace indexNs.

This allows us to clean up just like in notifyStartIndex and to set the index to a finished state so that the hooks are prevented from affecting it.

◆ notifyStartIndex()

void mongo::IndexConsistency::notifyStartIndex ( int indexNumber )

Informs the IndexConsistency instance that the index scan is beginning to scan the index with namespace indexNs.

This gives us a chance to clean up after the previous index and setup for the new index.

◆ relockCollectionWithMode()

void mongo::IndexConsistency::relockCollectionWithMode ( LockMode mode )

Gives up the lock that the collection is currently held in and requests the the collection again in LockMode mode

◆ removeDocKey()

void mongo::IndexConsistency::removeDocKey	(	const KeyString &	ks,
		int	indexNumber
	)

◆ removeIndexKey()

void mongo::IndexConsistency::removeIndexKey	(	const KeyString &	ks,
		int	indexNumber
	)

◆ scanLimitHit()

bool mongo::IndexConsistency::scanLimitHit ( )

Returns true if the ElapsedTracker says its time to yield during background validation.

◆ setLastProcessedIndexEntry()

void mongo::IndexConsistency::setLastProcessedIndexEntry	(	const IndexDescriptor &	descriptor,
		const boost::optional< IndexKeyEntry > &	indexEntry
	)

Sets _lastProcessedIndexEntry to the KeyString of indexEntry.

◆ setLastProcessedRecordId()

void mongo::IndexConsistency::setLastProcessedRecordId ( RecordId recordId )

Sets _lastProcessedRecordId to recordId.

◆ shouldGetNewSnapshot() [1/2]

bool mongo::IndexConsistency::shouldGetNewSnapshot ( const RecordId recordId ) const

Returns true if a new snapshot should be accquired.

If the recordId is equal to or greater than _yieldAtRecordId then we must get a new snapshot otherwise we will use stale data. Otherwise we do not need a new snapshot and can continue with the collection scan.

◆ shouldGetNewSnapshot() [2/2]

bool mongo::IndexConsistency::shouldGetNewSnapshot ( const KeyString & keyString ) const

Returns true if a new snapshot should be accquired.

If the keyString is equal to or greater than _yieldAtIndexEntry then we must get a new snapshot otherwise we will use stale data. Otherwise we do not need a new snapshot and can continue with the index scan.

Member Data Documentation

◆ _classMutex

stdx::mutex mongo::IndexConsistency::_classMutex

mutableprivate

◆ _collection

Collection* mongo::IndexConsistency::_collection

private

◆ _collLk

std::unique_ptr<Lock::CollectionLock> mongo::IndexConsistency::_collLk

private

◆ _currentIndex

int mongo::IndexConsistency::_currentIndex = -1

private

◆ _indexesInfo

std::map<int, IndexInfo> mongo::IndexConsistency::_indexesInfo

private

◆ _indexKeyCount

std::map<uint32_t, uint32_t> mongo::IndexConsistency::_indexKeyCount

private

◆ _indexNumber

std::map<std::string, int> mongo::IndexConsistency::_indexNumber

private

◆ _isBackground

const bool mongo::IndexConsistency::_isBackground

private

◆ _kErrorThreshold

const int mongo::IndexConsistency::_kErrorThreshold = 100

staticprivate

◆ _lastProcessedIndexEntry

std::unique_ptr<KeyString> mongo::IndexConsistency::_lastProcessedIndexEntry = nullptr

private

◆ _lastProcessedRecordId

boost::optional<RecordId> mongo::IndexConsistency::_lastProcessedRecordId = boost::none

private

◆ _nss

const NamespaceString mongo::IndexConsistency::_nss

private

◆ _numErrorsRecorded

int mongo::IndexConsistency::_numErrorsRecorded = 0

private

◆ _opCtx

OperationContext* mongo::IndexConsistency::_opCtx

private

◆ _recordStore

const RecordStore* mongo::IndexConsistency::_recordStore

private

◆ _stage

ValidationStage mongo::IndexConsistency::_stage = ValidationStage::DOCUMENT

private

◆ _tracker

ElapsedTracker mongo::IndexConsistency::_tracker

private

◆ _yieldAtIndexEntry

std::unique_ptr<KeyString> mongo::IndexConsistency::_yieldAtIndexEntry = nullptr

private

◆ _yieldAtRecordId

boost::optional<RecordId> mongo::IndexConsistency::_yieldAtRecordId = boost::none

private

The documentation for this class was generated from the following files:

/home/louis/git/mongodbsource.github.io/src/src/mongo/db/catalog/index_consistency.h
/home/louis/git/mongodbsource.github.io/src/src/mongo/db/catalog/index_consistency.cpp

Public Member Functions

Private Member Functions

Private Attributes

Static Private Attributes

Constructor & Destructor Documentation

◆ IndexConsistency()

Member Function Documentation

◆ _addDocKey_inlock()

◆ _addIndexKey_inlock()

◆ _hashKeyString()

◆ _isBeforeLastProcessedIndexEntry_inlock()

◆ _isBeforeLastProcessedRecordId_inlock()

◆ _isIndexFinished_inlock()

◆ _isIndexScanning_inlock()

◆ _removeDocKey_inlock()

◆ _removeIndexKey_inlock()

◆ _setYieldAtIndexEntry_inlock()

◆ _setYieldAtRecord_inlock()

◆ _throwExceptionIfError()

◆ addDocKey()

◆ addIndexKey()

◆ addLongIndexKey()

◆ applyChange()

◆ getIndexNumber()

◆ getNumExtraIndexKeys()

◆ getNumKeys()

◆ getNumLongKeys()

◆ getNumRecords()

◆ getStage()

◆ haveEntryMismatch()

◆ nextStage()

◆ notifyDoneIndex()

◆ notifyStartIndex()

◆ relockCollectionWithMode()

◆ removeDocKey()

◆ removeIndexKey()

◆ scanLimitHit()

◆ setLastProcessedIndexEntry()

◆ setLastProcessedRecordId()

◆ shouldGetNewSnapshot() [1/2]

◆ shouldGetNewSnapshot() [2/2]

Member Data Documentation

◆ _classMutex

◆ _collection

◆ _collLk

◆ _currentIndex

◆ _indexesInfo

◆ _indexKeyCount

◆ _indexNumber

◆ _isBackground

◆ _kErrorThreshold

◆ _lastProcessedIndexEntry

◆ _lastProcessedRecordId

◆ _nss

◆ _numErrorsRecorded

◆ _opCtx

◆ _recordStore

◆ _stage

◆ _tracker

◆ _yieldAtIndexEntry

◆ _yieldAtRecordId