diff --git a/src/index.js b/src/index.js index d81bc5e8..e495e4aa 100644 --- a/src/index.js +++ b/src/index.js @@ -122,7 +122,8 @@ export { intersectUpdateWithContentIds, intersectUpdateWithContentIdsV2, createDocFromUpdate, - createDocFromUpdateV2 + createDocFromUpdateV2, + MapConflictError } from './internals.js' export * from './utils/meta.js' diff --git a/src/internals.js b/src/internals.js index 207a74b7..1bc3be03 100644 --- a/src/internals.js +++ b/src/internals.js @@ -33,3 +33,4 @@ export * from './structs/ContentString.js' export * from './structs/ContentType.js' export * from './structs/Item.js' export * from './structs/Skip.js' +export * from './utils/MapConflict.js' diff --git a/src/structs/Item.js b/src/structs/Item.js index 3733348e..56f6e088 100644 --- a/src/structs/Item.js +++ b/src/structs/Item.js @@ -23,6 +23,7 @@ import { addChangedTypeToTransaction, addStructToIdSet, IdSet, StackItem, UpdateDecoderV1, UpdateDecoderV2, UpdateEncoderV1, UpdateEncoderV2, ContentType, ContentDeleted, StructStore, ID, YType, Transaction, // eslint-disable-line + recordMapWrite, parentIdString, classifyContent } from '../internals.js' import * as error from 'lib0/error' @@ -547,6 +548,31 @@ export class Item extends AbstractStruct { this.content.integrate(transaction, this) // add parent to transaction.changed addChangedTypeToTransaction(transaction, /** @type {YType} */ (this.parent), this.parentSub) + // record remote map set writes for conflict detection + if (!transaction.local && this.parentSub !== null && transaction.doc.mapConflictPolicy !== 'allow') { + const cls = classifyContent(this.content) + let val = null + let amb = cls.ambiguous + if (!amb) { + try { + const arr = this.content.getContent() + val = arr[arr.length - 1] + } catch (e) { + amb = true + } + } + recordMapWrite(transaction, /** @type {YType} */ (this.parent), this.parentSub, { + op: 'set', + key: this.parentSub, + parentId: parentIdString(/** @type {YType} */ (this.parent)), + client: this.id.client, + clock: this.id.clock, + source: 'remote', + ambiguous: amb, + contentKind: cls.kind, + value: val + }) + } if ((/** @type {YType} */ (this.parent)._item !== null && /** @type {YType} */ (this.parent)._item.deleted) || (this.parentSub !== null && this.right !== null)) { // delete if parent is deleted or if this is not the current attribute value of parent this.delete(transaction) diff --git a/src/utils/Doc.js b/src/utils/Doc.js index 4e8edf5e..f1192233 100644 --- a/src/utils/Doc.js +++ b/src/utils/Doc.js @@ -57,9 +57,19 @@ export class Doc extends ObservableV2 { /** * @param {DocOpts} opts configuration */ - constructor ({ guid = random.uuidv4(), collectionid = null, gc = true, gcFilter = () => true, meta = null, autoLoad = false, shouldLoad = true, isSuggestionDoc = false } = {}) { + constructor ({ guid = random.uuidv4(), collectionid = null, gc = true, gcFilter = () => true, meta = null, autoLoad = false, shouldLoad = true, isSuggestionDoc = false, mapConflictPolicy = 'allow' } = {}) { super() this.gc = gc + /** + * Map conflict policy: 'allow', 'collect', or 'error'. + * @type {'allow'|'collect'|'error'} + */ + this.mapConflictPolicy = mapConflictPolicy === 'collect' || mapConflictPolicy === 'error' ? mapConflictPolicy : 'allow' + /** + * Collected map conflicts (for 'collect' policy). + * @type {Array} + */ + this._mapConflicts = [] this.gcFilter = gcFilter this.clientID = generateNewClientId() this.guid = guid @@ -228,6 +238,46 @@ export class Doc extends ObservableV2 { return doc } + /** + * Returns the collected map conflicts. + * @return {Array} + */ + getMapConflicts () { + return this._mapConflicts.slice() + } + + /** + * Returns a summary of collected map conflicts. + * @return {{byType:Object, byKey:Object, byParent:Object, bySource:Object, count:number, total:number}} + */ + getMapConflictSummary () { + /** @type {Object} */ + const byType = Object.create(null) + /** @type {Object} */ + const byKey = Object.create(null) + /** @type {Object} */ + const byParent = Object.create(null) + /** @type {Object} */ + const bySource = Object.create(null) + let count = 0 + for (const c of this._mapConflicts) { + count++ + const t = c.type + byType[t] = (byType[t] || 0) + 1 + byKey[c.key] = (byKey[c.key] || 0) + 1 + byParent[c.parentId] = (byParent[c.parentId] || 0) + 1 + bySource[c.source] = (bySource[c.source] || 0) + 1 + } + return { byType, byKey, byParent, bySource, count, total: count } + } + + /** + * Clears the collected map conflicts. + */ + clearMapConflicts () { + this._mapConflicts = [] + } + /** * Emit `destroy` event and unregister all event handlers. */ diff --git a/src/utils/IdSet.js b/src/utils/IdSet.js index 7e73b87e..6214a6da 100644 --- a/src/utils/IdSet.js +++ b/src/utils/IdSet.js @@ -7,7 +7,8 @@ import { IdMap, AttrRanges, AttrRange, - Skip, AbstractStruct, IdSetDecoderV1, IdSetEncoderV1, IdSetDecoderV2, IdSetEncoderV2, Item, GC, StructStore, Transaction, ID // eslint-disable-line + Skip, AbstractStruct, IdSetDecoderV1, IdSetEncoderV1, IdSetDecoderV2, IdSetEncoderV2, Item, GC, StructStore, Transaction, ID, // eslint-disable-line + recordMapWrite, parentIdString, classifyContent } from '../internals.js' import * as array from 'lib0/array' @@ -774,6 +775,18 @@ export const readAndApplyDeleteSet = (decoder, transaction, store) => { // @ts-ignore struct = structs[index++] if (struct.id.clock < clockEnd) { + // record remote map delete writes for conflict detection (even if already deleted earlier in this tx) + if (struct instanceof Item && struct.parentSub !== null && transaction.doc.mapConflictPolicy !== 'allow') { + recordMapWrite(transaction, /** @type {any} */ (struct.parent), struct.parentSub, { + op: 'delete', + key: struct.parentSub, + parentId: parentIdString(/** @type {any} */ (struct.parent)), + client: struct.id.client, + clock: struct.id.clock, + source: transaction.local ? 'local' : 'remote', + ambiguous: false + }) + } if (!struct.deleted) { if (struct instanceof Item) { if (clockEnd < struct.id.clock + struct.length) { diff --git a/src/utils/MapConflict.js b/src/utils/MapConflict.js new file mode 100644 index 00000000..923bb397 --- /dev/null +++ b/src/utils/MapConflict.js @@ -0,0 +1,208 @@ +/** + * @module MapConflict + */ + +import { ContentType } from '../structs/ContentType.js' +import { ContentDoc } from '../structs/ContentDoc.js' +import { Item } from '../structs/Item.js' + +/** + * Error thrown when a map write conflict is detected in 'error' mode. + */ +export class MapConflictError extends Error { + /** + * @param {Array} conflicts + */ + constructor (conflicts) { + const msg = `MapConflictError: ${conflicts.length} conflict(s) detected on map key writes` + super(msg) + this.name = 'MapConflictError' + this.conflicts = conflicts + } +} + +/** + * Returns a summarized snapshot of a write (set/delete) for clear reporting. + * + * @param {Object} write + * @return {{summary:string}} + */ +const buildSnapshot = (write) => { + let summary + if (write.op === 'delete') { + summary = `delete key=${JSON.stringify(write.key)} parent=${write.parentId} by client=${write.client}@${write.clock}` + } else { + let valDesc + if (write.ambiguous) { + if (write.contentKind === 'YType') { + valDesc = '' + } else if (write.contentKind === 'YDoc') { + valDesc = '' + } else { + valDesc = '' + } + } else { + try { + const v = write.value + if (typeof v === 'string') valDesc = JSON.stringify(v) + else if (v === null || v === undefined) valDesc = String(v) + else if (typeof v === 'object') { + if (v instanceof Uint8Array) valDesc = `` + else valDesc = JSON.stringify(v) + } else valDesc = String(v) + } catch (e) { + valDesc = '' + } + } + summary = `set key=${JSON.stringify(write.key)}=${valDesc} parent=${write.parentId} by client=${write.client}@${write.clock}` + } + return { summary } +} + +/** + * Determines if a content value is ambiguous (YType or subdoc). + * + * @param {any} content + * @return {{ambiguous: boolean, kind: string|null}} + */ +export const classifyContent = (content) => { + if (content == null) return { ambiguous: false, kind: null } + if (content instanceof ContentType) return { ambiguous: true, kind: 'YType' } + if (content instanceof ContentDoc) return { ambiguous: true, kind: 'YDoc' } + return { ambiguous: false, kind: null } +} + +/** + * Records a write into the transaction's _mapWrites map. + * + * @param {import('./Transaction.js').Transaction} transaction + * @param {import('../ytype.js').YType} parent + * @param {string} key + * @param {Object} write + */ +export const recordMapWrite = (transaction, parent, key, write) => { + if (transaction.doc.mapConflictPolicy === 'allow') return + if (transaction._mapWrites == null) { + transaction._mapWrites = new Map() + } + let parentMap = transaction._mapWrites.get(parent) + if (parentMap == null) { + parentMap = new Map() + transaction._mapWrites.set(parent, parentMap) + } + let arr = parentMap.get(key) + if (arr == null) { + arr = [] + parentMap.set(key, arr) + } + arr.push(write) +} + +/** + * Returns a stable parent id string for reporting. + * + * @param {import('../ytype.js').YType} parent + */ +export const parentIdString = (parent) => { + if (parent == null) return 'null' + if (parent._item != null) { + const id = parent._item.id + return `${id.client}:${id.clock}` + } + // Top-level: look up the share key. + const doc = parent.doc + if (doc != null) { + for (const [k, v] of doc.share) { + if (v === parent) return `top:${k}` + } + } + return 'unknown' +} + +/** + * Compute conflicts from the recorded map writes in a transaction. + * + * @param {import('./Transaction.js').Transaction} transaction + * @return {Array} + */ +export const computeMapConflicts = (transaction) => { + /** @type {Array} */ + const conflicts = [] + if (transaction._mapWrites == null) return conflicts + const ownClient = transaction.doc.clientID + transaction._mapWrites.forEach((keyMap, parent) => { + keyMap.forEach((writes, key) => { + if (writes.length < 2) return + const sets = writes.filter(w => w.op === 'set') + const deletes = writes.filter(w => w.op === 'delete') + let type + if (sets.length >= 2 && deletes.length === 0) { + type = 'set-set' + } else if (deletes.length >= 1 && sets.length >= 1) { + type = 'delete-set' + } else if (sets.length === 1 && deletes.length === 0) { + // single set; ignore + return + } else if (deletes.length >= 2 && sets.length === 0) { + // multiple deletes; ignore (no real conflict; both produce empty) + return + } else { + type = 'set-set' + } + // ambiguous if any write involves YType or subdoc + const ambiguous = writes.some(w => w.ambiguous === true) + if (ambiguous) { + type = 'ambiguous' + } + // determine source + const localCount = writes.filter(w => w.source === 'local').length + const remoteCount = writes.length - localCount + let source + if (localCount === writes.length) source = 'local' + else if (remoteCount === writes.length) source = 'remote' + else source = 'mixed' + // determine winner deterministically: latest set wins (highest clock, then highest client), + // unless all are deletes. + const candidates = sets.length > 0 ? sets : writes + // sort by (clock, client) descending + const sorted = candidates.slice().sort((a, b) => { + if (b.clock !== a.clock) return b.clock - a.clock + return b.client - a.client + }) + const winnerWrite = sorted[0] + const winner = winnerWrite.op === 'set' + ? { op: 'set', value: winnerWrite.ambiguous ? null : winnerWrite.value, client: winnerWrite.client, clock: winnerWrite.clock, ambiguous: !!winnerWrite.ambiguous } + : { op: 'delete', client: winnerWrite.client, clock: winnerWrite.clock } + const parentId = writes[0].parentId + const writesOut = writes.map(w => ({ + op: w.op, + key, + parentId, + client: w.client, + clock: w.clock, + source: w.source, + ambiguous: !!w.ambiguous, + snapshot: buildSnapshot(w) + })) + const conflict = { + key, + parentId, + type, + source, + ambiguous, + message: `${type} conflict on key ${JSON.stringify(key)} of parent ${parentId} (writes: ${writes.length}, source: ${source}${ambiguous ? ', ambiguous' : ''})`, + writes: writesOut, + resolution: { + winner, + strategy: 'last-write-wins-by-id', + deterministic: true + } + } + // tag the conflict.type as ambiguous when ambiguous and also keep flag + conflicts.push(conflict) + }) + }) + // ensure ownClient is referenced (silence unused warning in strict mode) + void ownClient + return conflicts +} diff --git a/src/utils/Transaction.js b/src/utils/Transaction.js index d1a3a4e7..da131709 100644 --- a/src/utils/Transaction.js +++ b/src/utils/Transaction.js @@ -11,7 +11,8 @@ import { createID, iterateStructsByIdSet, ContentFormat, - IdSet, UpdateEncoderV1, UpdateEncoderV2, GC, StructStore, AbstractStruct, YEvent, Doc // eslint-disable-line + IdSet, UpdateEncoderV1, UpdateEncoderV2, GC, StructStore, AbstractStruct, YEvent, Doc, // eslint-disable-line + computeMapConflicts, MapConflictError } from '../internals.js' import { YType } from '../ytype.js' // eslint-disable-line @@ -130,6 +131,17 @@ export class Transaction { */ this._needFormattingCleanup = false this._done = false + /** + * Records map writes per (parent, key) for conflict detection. + * Null when no map conflict detection is enabled or no writes yet. + * @type {Map>>|null} + */ + this._mapWrites = null + /** + * The conflicts detected for this transaction, populated at finalization. + * @type {Array} + */ + this._mapConflicts = [] } /** @@ -505,6 +517,21 @@ const cleanupTransactions = (transactionCleanups, i) => { const store = doc.store const ds = transaction.deleteSet const mergeStructs = transaction._mergeStructs + // Map conflict detection + if (doc.mapConflictPolicy !== 'allow') { + const conflicts = computeMapConflicts(transaction) + if (conflicts.length > 0) { + transaction._mapConflicts = conflicts + if (doc.mapConflictPolicy === 'collect') { + for (const c of conflicts) doc._mapConflicts.push(c) + } + // In error mode, we throw after handling remaining cleanups via thrower + if (doc.mapConflictPolicy === 'error') { + // Continue with cleanup process but throw at the end + transaction._mapConflictError = new MapConflictError(conflicts) + } + } + } // insertIntoIdSet(store.ds, ds) try { doc.emit('beforeObserverCalls', [transaction, doc]) @@ -639,9 +666,11 @@ export const transact = (doc, f, origin = null, local = true) => { * @type {any} */ let result = null + let myTransaction = null if (doc._transaction === null) { initialCall = true doc._transaction = new Transaction(doc, origin, local) + myTransaction = doc._transaction transactionCleanups.push(doc._transaction) if (transactionCleanups.length === 1) { doc.emit('beforeAllTransactions', [doc]) @@ -667,5 +696,9 @@ export const transact = (doc, f, origin = null, local = true) => { } } } + // throw MapConflictError if conflicts were detected in error mode + if (myTransaction !== null && myTransaction._mapConflictError != null) { + throw myTransaction._mapConflictError + } return result } diff --git a/src/utils/encoding.js b/src/utils/encoding.js index 1cb6bf25..5f894805 100644 --- a/src/utils/encoding.js +++ b/src/utils/encoding.js @@ -36,6 +36,7 @@ import { readBlockSet, createIdSet, BlockSet, IdSet, IdSetDecoderV2, Doc, Transaction, GC, Item, StructStore, // eslint-disable-line + MapConflictError, createID, IdRange } from '../internals.js' @@ -452,6 +453,21 @@ export const readUpdate = (decoder, ydoc, transactionOrigin) => readUpdateV2(dec * @function */ export const applyUpdateV2 = (ydoc, update, transactionOrigin, YDecoder = UpdateDecoderV2) => { + // For 'error' policy, dry-run on a temporary doc to detect conflicts atomically. + if (ydoc.mapConflictPolicy === 'error' && !ydoc._mapConflictDryRun) { + const cloneDoc2 = new Doc({ guid: ydoc.guid, mapConflictPolicy: 'collect' }) + cloneDoc2._mapConflictDryRun = true + const stateV2 = encodeStateAsUpdateV2(ydoc) + const decBase = decoding.createDecoder(stateV2) + readUpdateV2(decBase, cloneDoc2, transactionOrigin, new UpdateDecoderV2(decBase)) + // clear any conflicts that arose from re-applying base state (shouldn't happen, but be safe) + cloneDoc2._mapConflicts = [] + const decUpd = decoding.createDecoder(update) + readUpdateV2(decUpd, cloneDoc2, transactionOrigin, new YDecoder(decUpd)) + if (cloneDoc2._mapConflicts.length > 0) { + throw new MapConflictError(cloneDoc2._mapConflicts) + } + } const decoder = decoding.createDecoder(update) readUpdateV2(decoder, ydoc, transactionOrigin, new YDecoder(decoder)) } diff --git a/src/ytype.js b/src/ytype.js index b90f4faa..600f69f5 100644 --- a/src/ytype.js +++ b/src/ytype.js @@ -19,7 +19,8 @@ import { getItemCleanStart, noAttributionsManager, transact, - ContentDoc, UpdateEncoderV1, UpdateEncoderV2, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, createAttributionFromAttributionItems, AbstractAttributionManager // eslint-disable-line + ContentDoc, UpdateEncoderV1, UpdateEncoderV2, Doc, Snapshot, Transaction, EventHandler, YEvent, Item, createAttributionFromAttributionItems, AbstractAttributionManager, // eslint-disable-line + recordMapWrite, parentIdString, classifyContent } from './internals.js' import * as contentType from './structs/ContentType.js' @@ -1740,6 +1741,18 @@ export const typeListDelete = (transaction, parent, index, length) => { export const typeMapDelete = (transaction, parent, key) => { const c = parent._map.get(key) if (c !== undefined) { + if (transaction.doc.mapConflictPolicy !== 'allow') { + const doc = transaction.doc + recordMapWrite(transaction, parent, key, { + op: 'delete', + key, + parentId: parentIdString(parent), + client: doc.clientID, + clock: getState(doc.store, doc.clientID), + source: transaction.local ? 'local' : 'remote', + ambiguous: false + }) + } c.delete(transaction) } } @@ -1785,7 +1798,22 @@ export const typeMapSet = (transaction, parent, key, value) => { } } } - new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, null, null, parent, key, content).integrate(transaction, 0) + const newItem = new Item(createID(ownClientId, getState(doc.store, ownClientId)), left, left && left.lastId, null, null, parent, key, content) + if (transaction.doc.mapConflictPolicy !== 'allow') { + const cls = classifyContent(content) + recordMapWrite(transaction, parent, key, { + op: 'set', + key, + parentId: parentIdString(parent), + client: newItem.id.client, + clock: newItem.id.clock, + source: transaction.local ? 'local' : 'remote', + ambiguous: cls.ambiguous, + contentKind: cls.kind, + value: cls.ambiguous ? null : value + }) + } + newItem.integrate(transaction, 0) } /**