# All files / local memory_mutation_queue.ts

100% Statements 185/185
95.65% Branches 44/46
100% Functions 26/26
100% Lines 179/179

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    114
    115
    116
    117
    118
    119
    120
    121
    122
    123
    124
    125
    126
    127
    128
    129
    130
    131
    132
    133
    134
    135
    136
    137
    138
    139
    140
    141
    142
    143
    144
    145
    146
    147
    148
    149
    150
    151
    152
    153
    154
    155
    156
    157
    158
    159
    160
    161
    162
    163
    164
    165
    166
    167
    168
    169
    170
    171
    172
    173
    174
    175
    176
    177
    178
    179
    180
    181
    182
    183
    184
    185
    186
    187
    188
    189
    190
    191
    192
    193
    194
    195
    196
    197
    198
    199
    200
    201
    202
    203
    204
    205
    206
    207
    208
    209
    210
    211
    212
    213
    214
    215
    216
    217
    218
    219
    220
    221
    222
    223
    224
    225
    226
    227
    228
    229
    230
    231
    232
    233
    234
    235
    236
    237
    238
    239
    240
    241
    242
    243
    244
    245
    246
    247
    248
    249
    250
    251
    252
    253
    254
    255
    256
    257
    258
    259
    260
    261
    262
    263
    264
    265
    266
    267
    268
    269
    270
    271
    272
    273
    274
    275
    276
    277
    278
    279
    280
    281
    282
    283
    284
    285
    286
    287
    288
    289
    290
    291
    292
    293
    294
    295
    296
    297
    298
    299
    300
    301
    302
    303
    304
    305
    306
    307
    308
    309
    310
    311
    312
    313
    314
    315
    316
    317
    318
    319
    320
    321
    322
    323
    324
    325
    326
    327
    328
    329
    330
    331
    332
    333
    334
    335
    336
    337
    338
    339
    340
    341
    342
    343
    344
    345
    346
    347
    348
    349
    350
    351
    352
    353
    354
    355
    356
    357
    358
    359
    360
    361
    362
    363
    364
    365
    366
    367
    368
    369
    370
    371
    372
    373
    374
    375
    376
    377
    378
    379
    380
    381
    382
    383
    384
    385
    386
    387
    388
    389
    390
    391
    392
    393
    394
    395
    396
    397
    398
    399
    400
    401
    402
    403
    404
    405
    406
    407
    408
    409
    410
    411
    412
    413
    414
    415
    416
    417
    418
    419
    420
    421
    422
    423
    424
    425
    426
    427
    428
    429
    430
    431
    432
    433
    434
    435
    436
    437
    438
    439
    440
    441
    442
    443
    444
    445
    446
    447
    448
    449
    450
    451
    452
    453
    454
    455
    456
    457
    458
    459
    460
    461
    462
    463
    464
    465
    466
    467
    468
    469
    470
    471
    472
    473
    474
    475
    476
    477
    478
    479
    480

    2x

    2x
    2x
    2x
    2x
    2x

    2x
    2x

    2x

    350x

    350x

    350x

    350x

    350x

    350x

    2x

    365x
    355x
    355x

    365x

    365x

    2x
    4x

    2x

    2x

    2x

    396x

    2x

    289x
    289x

    289x

    289x
    289x

    289x

    289x
    289x
    289x

    2x

    328x

    2x

    173x
    173x

    2x

    414x

    414x
    414x

    414x
    202x
    202x

    414x
    414x

    447x
    447x

    414x

    2x

    45x

    2x

    1243x

    1243x

    1243x
    1243x

    1243x
    342x
    342x
    338x

    905x

    2x

    30x

    2x

    13x

    13x
    13x
    1x
    12x
    1x

    11x

    13x

    1210x

    1210x
    1210x
    1210x
    1210x
    470x

    470x
    470x

    470x

    1210x

    131x

    131x
    131x

    131x
    131x
    131x

    131x

    131x

    131x
    21x
    21x
    2x

    19x
    16x

    19x

    131x
    131x
    14x
    14x
    14x

    131x

    2x

    335x
    335x

    335x
    335x

    335x
    335x

    335x
    335x
    335x
    17x
    17x
    3x
    3x

    14x

    14x
    14x

    335x
    319x
    139x
    139x
    135x

    319x
    319x

    16x
    16x

    335x
    349x
    349x
    367x
    367x
    367x
    339x

    367x
    367x

    335x
    335x

    2x
    348x

    2x

    456x
    456x
    456x

    2x

    315x
    149x

    315x

    2x

    43x

    43x
    118x
    118x
    98x

    43x

    2x
    624x
    624x

    624x

    2x
    2409x

    448x

    1961x
    1961x

    2x
    529x
    529x
    2x

    527x
    527x
    527x

    2x
     /**
     * Copyright 2017 Google Inc.
     *
     * Licensed under the Apache License, Version 2.0 (the "License");
     * you may not use this file except in compliance with the License.
     * You may obtain a copy of the License at
     *
     *   http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    import { Timestamp } from '../api/timestamp';
    import { Query } from '../core/query';
    import { BatchId, ProtoByteString } from '../core/types';
    import { DocumentKey } from '../model/document_key';
    import { Mutation } from '../model/mutation';
    import { BATCHID_UNKNOWN, MutationBatch } from '../model/mutation_batch';
    import { emptyByteString } from '../platform/platform';
    import { assert } from '../util/assert';
    import { primitiveComparator } from '../util/misc';
    import { SortedSet } from '../util/sorted_set';

    import { GarbageCollector } from './garbage_collector';
    import { MutationQueue } from './mutation_queue';
    import { PersistenceTransaction } from './persistence';
    import { PersistencePromise } from './persistence_promise';
    import { DocReference } from './reference_set';

    export class MemoryMutationQueue implements MutationQueue {
      /**
       * The set of all mutations that have been sent but not yet been applied to
       * the backend.
       */
      private mutationQueue: MutationBatch[] = [];

      /** Next value to use when assigning sequential IDs to each mutation batch. */
      private nextBatchId: BatchId = 1;

      /** The highest acknowledged mutation in the queue. */
      private highestAcknowledgedBatchId: BatchId = BATCHID_UNKNOWN;

      /** The last received stream token from the server, used to acknowledge which
       * responses the client has processed. Stream tokens are opaque checkpoint
       * markers whose only real value is their inclusion in the next request.
       */
      private lastStreamToken: ProtoByteString = emptyByteString();

      /** The garbage collector to notify about potential garbage keys. */
      private garbageCollector: GarbageCollector | null = null;

      /** An ordered mapping between documents and the mutations batch IDs. */
      private batchesByDocumentKey = new SortedSet(DocReference.compareByKey);

      start(transaction: PersistenceTransaction): PersistencePromise<void> {
        // NOTE: The queue may be shutdown / started multiple times, since we
        // maintain the queue for the duration of the app session in case a user
        // logs out / back in. To behave like the LevelDB-backed MutationQueue (and
        // accommodate tests that expect as much), we reset nextBatchId and
        // highestAcknowledgedBatchId if the queue is empty.
        if (this.mutationQueue.length === 0) {
          this.nextBatchId = 1;
          this.highestAcknowledgedBatchId = BATCHID_UNKNOWN;
        }
        assert(
          this.highestAcknowledgedBatchId < this.nextBatchId,
          'highestAcknowledgedBatchId must be less than the nextBatchId'
        );
        return PersistencePromise.resolve();
      }

      checkEmpty(transaction: PersistenceTransaction): PersistencePromise<boolean> {
        return PersistencePromise.resolve(this.mutationQueue.length === 0);
      }

      getNextBatchId(
        transaction: PersistenceTransaction
      ): PersistencePromise<BatchId> {
        return PersistencePromise.resolve(this.nextBatchId);
      }

      getHighestAcknowledgedBatchId(
        transaction: PersistenceTransaction
      ): PersistencePromise<BatchId> {
        return PersistencePromise.resolve(this.highestAcknowledgedBatchId);
      }

      acknowledgeBatch(
        transaction: PersistenceTransaction,
        batch: MutationBatch,
        streamToken: ProtoByteString
      ): PersistencePromise<void> {
        const batchId = batch.batchId;
        assert(
          batchId > this.highestAcknowledgedBatchId,
          'Mutation batchIDs must be acknowledged in order'
        );

        const batchIndex = this.indexOfExistingBatchId(batchId, 'acknowledged');

        // Verify that the batch in the queue is the one to be acknowledged.
        const check = this.mutationQueue[batchIndex];
        assert(
          batchId === check.batchId,
          'Queue ordering failure: expected batch ' +
            batchId +
            ', got batch ' +
            check.batchId
        );
        assert(
          !check.isTombstone(),
          "Can't acknowledge a previously removed batch"
        );

        this.highestAcknowledgedBatchId = batchId;
        this.lastStreamToken = streamToken;
        return PersistencePromise.resolve();
      }

      getLastStreamToken(
        transaction: PersistenceTransaction
      ): PersistencePromise<ProtoByteString> {
        return PersistencePromise.resolve(this.lastStreamToken);
      }

      setLastStreamToken(
        transaction: PersistenceTransaction,
        streamToken: ProtoByteString
      ): PersistencePromise<void> {
        this.lastStreamToken = streamToken;
        return PersistencePromise.resolve();
      }

      addMutationBatch(
        transaction: PersistenceTransaction,
        localWriteTime: Timestamp,
        mutations: Mutation[]
      ): PersistencePromise<MutationBatch> {
        assert(mutations.length !== 0, 'Mutation batches should not be empty');

        const batchId = this.nextBatchId;
        this.nextBatchId++;

        if (this.mutationQueue.length > 0) {
          const prior = this.mutationQueue[this.mutationQueue.length - 1];
          assert(
            prior.batchId < batchId,
            'Mutation batchIDs must be monotonically increasing order'
          );
        }

        const batch = new MutationBatch(batchId, localWriteTime, mutations);
        this.mutationQueue.push(batch);

        // Track references by document key.
        for (const mutation of mutations) {
          this.batchesByDocumentKey = this.batchesByDocumentKey.add(
            new DocReference(mutation.key, batchId)
          );
        }

        return PersistencePromise.resolve(batch);
      }

      lookupMutationBatch(
        transaction: PersistenceTransaction,
        batchId: BatchId
      ): PersistencePromise<MutationBatch | null> {
        return PersistencePromise.resolve(this.findMutationBatch(batchId));
      }

      getNextMutationBatchAfterBatchId(
        transaction: PersistenceTransaction,
        batchId: BatchId
      ): PersistencePromise<MutationBatch | null> {
        const size = this.mutationQueue.length;

        // All batches with batchId <= this.highestAcknowledgedBatchId have been
        // acknowledged so the first unacknowledged batch after batchID will have a
        // batchID larger than both of these values.
        const nextBatchId = Math.max(batchId, this.highestAcknowledgedBatchId) + 1;

        // The requested batchId may still be out of range so normalize it to the
        // start of the queue.
        const rawIndex = this.indexOfBatchId(nextBatchId);
        let index = rawIndex < 0 ? 0 : rawIndex;

        // Finally return the first non-tombstone batch.
        for (; index < size; index++) {
          const batch = this.mutationQueue[index];
          if (!batch.isTombstone()) {
            return PersistencePromise.resolve(batch);
          }
        }
        return PersistencePromise.resolve(null);
      }

      getAllMutationBatches(
        transaction: PersistenceTransaction
      ): PersistencePromise<MutationBatch[]> {
        return PersistencePromise.resolve(
          this.getAllLiveMutationBatchesBeforeIndex(this.mutationQueue.length)
        );
      }

      getAllMutationBatchesThroughBatchId(
        transaction: PersistenceTransaction,
        batchId: BatchId
      ): PersistencePromise<MutationBatch[]> {
        const count = this.mutationQueue.length;

        let endIndex = this.indexOfBatchId(batchId);
        if (endIndex < 0) {
          endIndex = 0;
        } else if (endIndex >= count) {
          endIndex = count;
        } else {
          // The endIndex is in the queue so increment to pull everything in the
          // queue including it.
          endIndex++;
        }

        return PersistencePromise.resolve(
          this.getAllLiveMutationBatchesBeforeIndex(endIndex)
        );
      }

      getAllMutationBatchesAffectingDocumentKey(
        transaction: PersistenceTransaction,
        documentKey: DocumentKey
      ): PersistencePromise<MutationBatch[]> {
        const start = new DocReference(documentKey, 0);
        const end = new DocReference(documentKey, Number.POSITIVE_INFINITY);
        const result: MutationBatch[] = [];
        this.batchesByDocumentKey.forEachInRange([start, end], ref => {
          assert(
            documentKey.isEqual(ref.key),
            "Should only iterate over a single key's batches"
          );
          const batch = this.findMutationBatch(ref.targetOrBatchId);
          assert(
            batch !== null,
            'Batches in the index must exist in the main table'
          );
          result.push(batch!);
        });

        return PersistencePromise.resolve(result);
      }

      getAllMutationBatchesAffectingQuery(
        transaction: PersistenceTransaction,
        query: Query
      ): PersistencePromise<MutationBatch[]> {
        // Use the query path as a prefix for testing if a document matches the
        // query.
        const prefix = query.path;
        const immediateChildrenPathLength = prefix.length + 1;

        // Construct a document reference for actually scanning the index. Unlike
        // the prefix the document key in this reference must have an even number of
        // segments. The empty segment can be used a suffix of the query path
        // because it precedes all other segments in an ordered traversal.
        let startPath = prefix;
        Eif (!DocumentKey.isDocumentKey(startPath)) {
          startPath = startPath.child('');
        }

        const start = new DocReference(new DocumentKey(startPath), 0);

        // Find unique batchIDs referenced by all documents potentially matching the
        // query.
        let uniqueBatchIDs = new SortedSet<number>(primitiveComparator);

        this.batchesByDocumentKey.forEachWhile(ref => {
          const rowKeyPath = ref.key.path;
          if (!prefix.isPrefixOf(rowKeyPath)) {
            return false;
          } else {
            // Rows with document keys more than one segment longer than the query
            // path can't be matches. For example, a query on 'rooms' can't match
            // the document /rooms/abc/messages/xyx.
            // TODO(mcg): we'll need a different scanner when we implement
            // ancestor queries.
            if (rowKeyPath.length === immediateChildrenPathLength) {
              uniqueBatchIDs = uniqueBatchIDs.add(ref.targetOrBatchId);
            }
            return true;
          }
        }, start);

        // Construct an array of matching batches, sorted by batchID to ensure that
        // multiple mutations affecting the same document key are applied in order.
        const result: MutationBatch[] = [];
        uniqueBatchIDs.forEach(batchId => {
          const batch = this.findMutationBatch(batchId);
          Eif (batch !== null) {
            result.push(batch);
          }
        });
        return PersistencePromise.resolve(result);
      }

      removeMutationBatches(
        transaction: PersistenceTransaction,
        batches: MutationBatch[]
      ): PersistencePromise<void> {
        const batchCount = batches.length;
        assert(batchCount > 0, 'Should not remove mutations when none exist.');

        const firstBatchId = batches[0].batchId;
        const queueCount = this.mutationQueue.length;

        // Find the position of the first batch for removal. This need not be the
        // first entry in the queue.
        const startIndex = this.indexOfExistingBatchId(firstBatchId, 'removed');
        assert(
          this.mutationQueue[startIndex].batchId === firstBatchId,
          'Removed batches must exist in the queue'
        );

        // Check that removed batches are contiguous (while excluding tombstones).
        let batchIndex = 1;
        let queueIndex = startIndex + 1;
        while (batchIndex < batchCount && queueIndex < queueCount) {
          const batch = this.mutationQueue[queueIndex];
          if (batch.isTombstone()) {
            queueIndex++;
            continue;
          }

          assert(
            batch.batchId === batches[batchIndex].batchId,
            'Removed batches must be contiguous in the queue'
          );
          batchIndex++;
          queueIndex++;
        }

        // Only actually remove batches if removing at the front of the queue.
        // Previously rejected batches may have left tombstones in the queue, so
        // expand the removal range to include any tombstones.
        if (startIndex === 0) {
          for (; queueIndex < queueCount; queueIndex++) {
            const batch = this.mutationQueue[queueIndex];
            if (!batch.isTombstone()) {
              break;
            }
          }
          const length = queueIndex - startIndex;
          this.mutationQueue.splice(startIndex, length);
        } else {
          // Mark the tombstones
          for (let i = startIndex; i < queueIndex; i++) {
            this.mutationQueue[i] = this.mutationQueue[i].toTombstone();
          }
        }

        let references = this.batchesByDocumentKey;
        for (const batch of batches) {
          const batchId = batch.batchId;
          for (const mutation of batch.mutations) {
            const key = mutation.key;
            if (this.garbageCollector !== null) {
              this.garbageCollector.addPotentialGarbageKey(key);
            }

            const ref = new DocReference(key, batchId);
            references = references.delete(ref);
          }
        }
        this.batchesByDocumentKey = references;
        return PersistencePromise.resolve();
      }

      setGarbageCollector(garbageCollector: GarbageCollector | null): void {
        this.garbageCollector = garbageCollector;
      }

      containsKey(
        txn: PersistenceTransaction,
        key: DocumentKey
      ): PersistencePromise<boolean> {
        const ref = new DocReference(key, 0);
        const firstRef = this.batchesByDocumentKey.firstAfterOrEqual(ref);
        return PersistencePromise.resolve(key.isEqual(firstRef && firstRef.key));
      }

      performConsistencyCheck(
        txn: PersistenceTransaction
      ): PersistencePromise<void> {
        if (this.mutationQueue.length === 0) {
          assert(
            this.batchesByDocumentKey.isEmpty(),
            'Document leak -- detected dangling mutation references when queue is empty.'
          );
        }
        return PersistencePromise.resolve();
      }

      /**
       * A private helper that collects all the mutations batches in the queue up to
       * but not including the given endIndex. All tombstones in the queue are
       * excluded.
       */
      private getAllLiveMutationBatchesBeforeIndex(
        endIndex: number
      ): MutationBatch[] {
        const result: MutationBatch[] = [];

        for (let i = 0; i < endIndex; i++) {
          const batch = this.mutationQueue[i];
          if (!batch.isTombstone()) {
            result.push(batch);
          }
        }

        return result;
      }

      /**
       * Finds the index of the given batchId in the mutation queue and asserts that
       * the resulting index is within the bounds of the queue.
       *
       * @param batchId The batchId to search for
       * @param action A description of what the caller is doing, phrased in passive
       * form (e.g. "acknowledged" in a routine that acknowledges batches).
       */
      private indexOfExistingBatchId(batchId: BatchId, action: string): number {
        const index = this.indexOfBatchId(batchId);
        assert(
          index >= 0 && index < this.mutationQueue.length,
          'Batches must exist to be ' + action
        );
        return index;
      }

      /**
       * Finds the index of the given batchId in the mutation queue. This operation
       * is O(1).
       *
       * @return The computed index of the batch with the given batchId, based on
       * the state of the queue. Note this index can be negative if the requested
       * batchId has already been remvoed from the queue or past the end of the
       * queue if the batchId is larger than the last added batch.
       */
      private indexOfBatchId(batchId: BatchId): number {
        if (this.mutationQueue.length === 0) {
          // As an index this is past the end of the queue
          return 0;
        }

        // Examine the front of the queue to figure out the difference between the
        // batchId and indexes in the array. Note that since the queue is ordered
        // by batchId, if the first batch has a larger batchId then the requested
        // batchId doesn't exist in the queue.
        const firstBatchId = this.mutationQueue[0].batchId;
        return batchId - firstBatchId;
      }

      /**
       * A version of lookupMutationBatch that doesn't return a promise, this makes
       * other functions that uses this code easier to read and more efficent.
       */
      private findMutationBatch(batchId: BatchId): MutationBatch | null {
        const index = this.indexOfBatchId(batchId);
        if (index < 0 || index >= this.mutationQueue.length) {
          return null;
        }

        const batch = this.mutationQueue[index];
        assert(batch.batchId === batchId, 'If found batch must match');
        return batch.isTombstone() ? null : batch;
      }
    }