index_consistency.h

Go to the documentation of this file.

#pragma once

#include "mongo/db/concurrency/d_concurrency.h"
#include "mongo/db/index/index_descriptor.h"
#include "mongo/db/storage/key_string.h"
#include "mongo/db/storage/record_store.h"
#include "mongo/db/storage/sorted_data_interface.h"
#include "mongo/util/elapsed_tracker.h"

namespace mongo {

enum class ValidationStage { DOCUMENT, INDEX, NONE };

enum class ValidationOperation { INSERT, REMOVE };

struct IndexInfo {
    // Informs us if the index was ready or not for consumption during the start of validation.
    bool isReady;
    // Contains the pre-computed hashed of the index namespace.
    uint32_t indexNsHash;
    // True if the index has finished scanning from the index scan stage, otherwise false.
    bool indexScanFinished;
    // The number of index entries belonging to the index.
    int64_t numKeys;
    // The number of long keys that are not indexed for the index.
    int64_t numLongKeys;
    // The number of records that have a key in their document that referenced back to the
    // this index
    int64_t numRecords;
    // Keeps track of how many indexes were removed (-1) and added (+1) after the
    // point of validity was set for this index.
    int64_t numExtraIndexKeys;
};

class IndexConsistency final {
public:
    IndexConsistency(OperationContext* opCtx,
                     Collection* collection,
                     NamespaceString nss,
                     RecordStore* recordStore,
                     std::unique_ptr<Lock::CollectionLock> collLk,
                     const bool background);

    void addDocKey(const KeyString& ks, int indexNumber);
    void removeDocKey(const KeyString& ks, int indexNumber);
    void addIndexKey(const KeyString& ks, int indexNumber);
    void removeIndexKey(const KeyString& ks, int indexNumber);

    void addLongIndexKey(int indexNumber);

    int64_t getNumKeys(int indexNumber) const;

    int64_t getNumLongKeys(int indexNumber) const;

    int64_t getNumRecords(int indexNumber) const;

    bool haveEntryMismatch() const;

    int64_t getNumExtraIndexKeys(int indexNumber) const;

    void applyChange(const IndexDescriptor* descriptor,
                     const boost::optional<IndexKeyEntry>& indexEntry,
                     ValidationOperation operation);

    void nextStage();

    ValidationStage getStage() const;

    void setLastProcessedRecordId(RecordId recordId);

    void setLastProcessedIndexEntry(const IndexDescriptor& descriptor,
                                    const boost::optional<IndexKeyEntry>& indexEntry);

    void notifyStartIndex(int indexNumber);

    void notifyDoneIndex(int indexNumber);

    int getIndexNumber(const std::string& indexNs);

    bool shouldGetNewSnapshot(const RecordId recordId) const;

    bool shouldGetNewSnapshot(const KeyString& keyString) const;

    void relockCollectionWithMode(LockMode mode);

    bool scanLimitHit();

private:
    OperationContext* _opCtx;
    Collection* _collection;
    const NamespaceString _nss;
    const RecordStore* _recordStore;
    std::unique_ptr<Lock::CollectionLock> _collLk;
    const bool _isBackground;
    ElapsedTracker _tracker;

    // We map the hashed KeyString values to a bucket which contain the count of how many
    // index keys and document keys we've seen in each bucket.
    // Count rules:
    //     - If the count is 0 in the bucket, we have index consistency for
    //       KeyStrings that mapped to it
    //     - If the count is > 0 in the bucket at the end of the validation pass, then there
    //       are too few index entries.
    //     - If the count is < 0 in the bucket at the end of the validation pass, then there
    //       are too many index entries.
    std::map<uint32_t, uint32_t> _indexKeyCount;

    // Contains the corresponding index number for each index namespace
    std::map<std::string, int> _indexNumber;

    // A mapping of index numbers to IndexInfo
    std::map<int, IndexInfo> _indexesInfo;

    // RecordId of the last processed document during the collection scan.
    boost::optional<RecordId> _lastProcessedRecordId = boost::none;

    // KeyString of the last processed index entry during the index scan.
    std::unique_ptr<KeyString> _lastProcessedIndexEntry = nullptr;

    // The current index namespace being scanned in the index scan phase.
    int _currentIndex = -1;

    // The stage that the validation is currently on.
    ValidationStage _stage = ValidationStage::DOCUMENT;

    // Contains the RecordId of when we should yield collection scan.
    boost::optional<RecordId> _yieldAtRecordId = boost::none;

    // Contains the KeyString of when we should yield during the index scan.
    std::unique_ptr<KeyString> _yieldAtIndexEntry = nullptr;

    // Threshold for the number of errors to record before returning "There are too many errors".
    static const int _kErrorThreshold = 100;

    // The current number of errors that are recorded.
    int _numErrorsRecorded = 0;

    // Only one thread can use the class at a time
    mutable stdx::mutex _classMutex;

    void _addDocKey_inlock(const KeyString& ks, int indexNumber);

    void _removeDocKey_inlock(const KeyString& ks, int indexNumber);

    void _addIndexKey_inlock(const KeyString& ks, int indexNumber);

    void _removeIndexKey_inlock(const KeyString& ks, int indexNumber);

    bool _isIndexFinished_inlock(int indexNumber) const;

    bool _isIndexScanning_inlock(int indexNumber) const;

    void _setYieldAtRecord_inlock(const RecordId recordId);

    void _setYieldAtIndexEntry_inlock(const KeyString& keyString);

    bool _isBeforeLastProcessedRecordId_inlock(RecordId recordId) const;

    bool _isBeforeLastProcessedIndexEntry_inlock(const KeyString& keyString) const;

    uint32_t _hashKeyString(const KeyString& ks, int indexNumbers) const;

    Status _throwExceptionIfError();

};  // IndexConsistency
}  // namespace mongo