From fd723efcfa908adb637c47f05e8f4d43bd35f40a Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Tue, 6 Dec 2022 16:51:02 -0500 Subject: [PATCH] added graphene find path tool using l2cache when available for fast and more accurate find path check for l2cache existing is now cached and only checked when necessary --- .../datasource/graphene/frontend.ts | 1086 ++++++++++++----- .../datasource/graphene/graphene.css | 39 +- src/neuroglancer/status.ts | 1 + src/neuroglancer/ui/annotations.ts | 239 ++-- src/neuroglancer/util/json.ts | 10 + 5 files changed, 940 insertions(+), 435 deletions(-) diff --git a/src/neuroglancer/datasource/graphene/frontend.ts b/src/neuroglancer/datasource/graphene/frontend.ts index 251c216dd3..f35a7e0a81 100644 --- a/src/neuroglancer/datasource/graphene/frontend.ts +++ b/src/neuroglancer/datasource/graphene/frontend.ts @@ -16,8 +16,9 @@ import './graphene.css'; -import {AnnotationReference, AnnotationType, Line, LocalAnnotationSource, makeDataBoundsBoundingBoxAnnotationSet, Point} from 'neuroglancer/annotation'; +import {Annotation, AnnotationReference, AnnotationSource, AnnotationType, Line, LocalAnnotationSource, makeDataBoundsBoundingBoxAnnotationSet, Point} from 'neuroglancer/annotation'; import {AnnotationDisplayState, AnnotationLayerState} from 'neuroglancer/annotation/annotation_layer_state'; +import {MultiscaleAnnotationSource} from 'neuroglancer/annotation/frontend_source'; import {LayerChunkProgressInfo} from 'neuroglancer/chunk_manager/base'; import {ChunkManager, WithParameters} from 'neuroglancer/chunk_manager/frontend'; import {makeIdentityTransform} from 'neuroglancer/coordinate_transform'; @@ -43,16 +44,19 @@ import {SliceViewPanelRenderLayer, SliceViewRenderLayer} from 'neuroglancer/slic import {StatusMessage} from 'neuroglancer/status'; import {TrackableBoolean, TrackableBooleanCheckbox} from 'neuroglancer/trackable_boolean'; import {makeCachedLazyDerivedWatchableValue, NestedStateManager, registerNested, TrackableValue, WatchableSet, WatchableValue, WatchableValueInterface} from 'neuroglancer/trackable_value'; -import {AnnotationLayerView, MergedAnnotationStates, PlaceLineTool} from 'neuroglancer/ui/annotations'; +import {AnnotationLayerView, makeAnnotationListElement, MergedAnnotationStates, PlaceLineTool} from 'neuroglancer/ui/annotations'; +import {getDefaultAnnotationListBindings} from 'neuroglancer/ui/default_input_event_bindings'; import {LayerTool, makeToolActivationStatusMessageWithHeader, makeToolButton, registerLegacyTool, registerTool, ToolActivation} from 'neuroglancer/ui/tool'; import {Uint64Set} from 'neuroglancer/uint64_set'; import {packColor} from 'neuroglancer/util/color'; import {Owned, RefCounted} from 'neuroglancer/util/disposable'; +import {removeChildren} from 'neuroglancer/util/dom'; import {makeValueOrError, ValueOrError, valueOrThrow} from 'neuroglancer/util/error'; import {EventActionMap} from 'neuroglancer/util/event_action_map'; import {mat4, vec3, vec4} from 'neuroglancer/util/geom'; import {HttpError, isNotFoundError, responseJson} from 'neuroglancer/util/http_request'; -import {parseArray, parseFixedLengthArray, verify3dVec, verifyBoolean, verifyEnumString, verifyFiniteFloat, verifyFinitePositiveFloat, verifyInt, verifyNonnegativeInt, verifyObject, verifyObjectProperty, verifyOptionalObjectProperty, verifyOptionalString, verifyPositiveInt, verifyString} from 'neuroglancer/util/json'; +import {parseArray, parseFixedLengthArray, verify3dVec, verifyBoolean, verifyEnumString, verifyFiniteFloat, verifyFinitePositiveFloat, verifyFloatArray, verifyInt, verifyIntegerArray, verifyNonnegativeInt, verifyObject, verifyObjectProperty, verifyOptionalObjectProperty, verifyOptionalString, verifyPositiveInt, verifyString, verifyStringArray} from 'neuroglancer/util/json'; +import {MouseEventBinder} from 'neuroglancer/util/mouse_bindings'; import {getObjectId} from 'neuroglancer/util/object_id'; import {NullarySignal} from 'neuroglancer/util/signal'; import {cancellableFetchSpecialOk, parseSpecialUrl, SpecialProtocolCredentials, SpecialProtocolCredentialsProvider} from 'neuroglancer/util/special_protocol_request'; @@ -79,6 +83,7 @@ const RED_COLOR_SEGMENT_PACKED = new Uint64(packColor(RED_COLOR_SEGMENT)); const BLUE_COLOR_SEGMENT_PACKED = new Uint64(packColor(BLUE_COLOR_SEGMENT)); const TRANSPARENT_COLOR_PACKED = new Uint64(packColor(TRANSPARENT_COLOR)); const MULTICUT_OFF_COLOR = vec4.fromValues(0, 0, 0, 0.5); +const WHITE_COLOR = vec3.fromValues(1, 1, 1); class GrapheneMeshSource extends (WithParameters(WithCredentialsProvider()(MeshSource), MeshSourceParameters)) { @@ -91,6 +96,8 @@ class GrapheneMeshSource extends class AppInfo { segmentationUrl: string; meshingUrl: string; + l2CacheUrl: string; + table: string; supported_api_versions: number[]; constructor(infoUrl: string, obj: any) { // .../1.0/... is the legacy link style @@ -100,8 +107,11 @@ class AppInfo { if (match === null) { throw Error(`Graph URL invalid: ${infoUrl}`); } - this.segmentationUrl = `${match[1]}/segmentation/api/v${PYCG_APP_VERSION}/table/${match[2]}`; - this.meshingUrl = `${match[1]}/meshing/api/v${PYCG_APP_VERSION}/table/${match[2]}`; + this.table = match[2]; + const {table} = this; + this.segmentationUrl = `${match[1]}/segmentation/api/v${PYCG_APP_VERSION}/table/${table}`; + this.meshingUrl = `${match[1]}/meshing/api/v${PYCG_APP_VERSION}/table/${table}`; + this.l2CacheUrl = `${match[1]}/l2cache/api/v${PYCG_APP_VERSION}`; try { verifyObject(obj); @@ -352,7 +362,7 @@ async function getVolumeDataSource( const info = parseGrapheneMultiscaleVolumeInfo(metadata, url, options.credentialsManager); const volume = new GrapheneMultiscaleVolumeChunkSource(options.chunkManager, credentialsProvider, info); - const state = new GrapheneState() + const state = new GrapheneState(); if (options.state) { state.restoreState(options.state) } @@ -462,7 +472,8 @@ function makeColoredAnnotationState( layer: SegmentationUserLayer, loadedSubsource: LoadedDataSubsource, subsubsourceId: string, color: vec3) { const {subsourceEntry} = loadedSubsource; - const source = new LocalAnnotationSource(loadedSubsource.loadedDataSource.transform, [], ['associated segments']); + const source = new LocalAnnotationSource( + loadedSubsource.loadedDataSource.transform, [], ['associated segments']); const displayState = new AnnotationDisplayState(); displayState.color.value.set(color); @@ -498,59 +509,86 @@ function getUint64(obj: any, key: string) { function restoreSegmentSelection(obj: any): SegmentSelection { const segmentId = getUint64(obj, SEGMENT_ID_JSON_KEY); const rootId = getUint64(obj, ROOT_ID_JSON_KEY); - const position = verifyObjectProperty( - obj, POSITION_JSON_KEY, value => { - return verify3dVec(value); - }); + const position = verifyObjectProperty(obj, POSITION_JSON_KEY, value => { + return verify3dVec(value); + }); return { segmentId, rootId, position, - } + }; } +const segmentSelectionToJSON = (x: SegmentSelection) => { + return { + [SEGMENT_ID_JSON_KEY]: x.segmentId.toJSON(), + [ROOT_ID_JSON_KEY]: x.rootId.toJSON(), + [POSITION_JSON_KEY]: [...x.position], + }; +}; + const ID_JSON_KEY = 'id'; -const ERROR_JSON_KEY = 'error'; +const SEGMENT_ID_JSON_KEY = 'segmentId'; +const ROOT_ID_JSON_KEY = 'rootId'; +const POSITION_JSON_KEY = 'position'; +const SINK_JSON_KEY = 'sink'; +const SOURCE_JSON_KEY = 'source'; + const MULTICUT_JSON_KEY = 'multicut'; const FOCUS_SEGMENT_JSON_KEY = 'focusSegment'; const SINKS_JSON_KEY = 'sinks'; const SOURCES_JSON_KEY = 'sources'; -const SEGMENT_ID_JSON_KEY = 'segmentId'; -const ROOT_ID_JSON_KEY = 'rootId'; -const POSITION_JSON_KEY = 'position'; + const MERGE_JSON_KEY = 'merge'; const MERGES_JSON_KEY = 'merges'; const AUTOSUBMIT_JSON_KEY = 'autosubmit'; -const SINK_JSON_KEY = 'sink'; -const SOURCE_JSON_KEY = 'source'; -const MERGED_ROOT_JSON_KEY = 'mergedRoot'; const LOCKED_JSON_KEY = 'locked'; +const MERGED_ROOT_JSON_KEY = 'mergedRoot'; +const ERROR_JSON_KEY = 'error'; + +const FIND_PATH_JSON_KEY = 'findPath'; +const TARGET_JSON_KEY = 'target'; +const CENTROIDS_JSON_KEY = 'centroids'; +const PRECISION_MODE_JSON_KEY = 'precision'; -class GrapheneState implements Trackable { + + +class GrapheneState extends RefCounted implements Trackable { changed = new NullarySignal(); public multicutState = new MulticutState(); public mergeState = new MergeState(); + public findPathState = new FindPathState(); constructor() { - this.multicutState.changed.add(() => { + super(); + this.registerDisposer(this.multicutState.changed.add(() => { this.changed.dispatch(); - }); - this.mergeState.changed.add(() => { + })); + this.registerDisposer(this.mergeState.changed.add(() => { this.changed.dispatch(); - }); + })); + this.registerDisposer(this.findPathState.changed.add(() => {this.changed.dispatch()})); + } + + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + this.multicutState.replaceSegments(oldValues, newValues); + this.mergeState.replaceSegments(oldValues, newValues); + this.findPathState.replaceSegments(oldValues, newValues); } reset() { this.multicutState.reset(); this.mergeState.reset(); + this.findPathState.reset(); } toJSON() { return { [MULTICUT_JSON_KEY]: this.multicutState.toJSON(), [MERGE_JSON_KEY]: this.mergeState.toJSON(), - } + [FIND_PATH_JSON_KEY]: this.findPathState.toJSON(), + }; } restoreState(x: any) { @@ -560,6 +598,9 @@ class GrapheneState implements Trackable { verifyOptionalObjectProperty(x, MERGE_JSON_KEY, value => { this.mergeState.restoreState(value); }); + verifyOptionalObjectProperty(x, FIND_PATH_JSON_KEY, value => { + this.findPathState.restoreState(value); + }); } } @@ -581,6 +622,29 @@ class MergeState extends RefCounted implements Trackable { this.registerDisposer(this.merges.changed.add(this.changed.dispatch)); } + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + const {merges: {value: merges}} = this; + const newValue = newValues.size === 1 ? [...newValues][0] : undefined; + for (const merge of merges) { + if (merge.source && oldValues.has(merge.source.rootId)) { + if (newValue) { + merge.source.rootId = newValue; + } else { + this.reset(); + return; + } + } + if (merge.sink && oldValues.has(merge.sink.rootId)) { + if (newValue) { + merge.sink.rootId = newValue; + } else { + this.reset(); + return; + } + } + } + } + reset() { this.merges.value = []; this.autoSubmit.reset(); @@ -588,35 +652,23 @@ class MergeState extends RefCounted implements Trackable { toJSON() { const {merges, autoSubmit} = this; - - const segmentSelectionToJSON = (x: SegmentSelection) => { - return { - [SEGMENT_ID_JSON_KEY]: x.segmentId.toJSON(), - [ROOT_ID_JSON_KEY]: x.rootId.toJSON(), - [POSITION_JSON_KEY]: [...x.position], - } - } - const mergeToJSON = (x: MergeSubmission) => { const res: any = { [ID_JSON_KEY]: x.id, [LOCKED_JSON_KEY]: x.locked, [SINK_JSON_KEY]: segmentSelectionToJSON(x.sink), [SOURCE_JSON_KEY]: segmentSelectionToJSON(x.source!), - } - + }; if (x.mergedRoot) { res[MERGED_ROOT_JSON_KEY] = x.mergedRoot.toJSON(); } if (x.error) { res[ERROR_JSON_KEY] = x.error; } - return res; - } - + }; return { - [MERGES_JSON_KEY]: merges.value.filter(x=>x.source).map(mergeToJSON), + [MERGES_JSON_KEY]: merges.value.filter(x => x.source).map(mergeToJSON), [AUTOSUBMIT_JSON_KEY]: autoSubmit.toJSON(), }; } @@ -636,7 +688,7 @@ class MergeState extends RefCounted implements Trackable { source, mergedRoot, error, - } + }; } const submissionsValidator = (value: any) => { @@ -646,7 +698,133 @@ class MergeState extends RefCounted implements Trackable { }; this.merges.value = verifyObjectProperty(x, MERGES_JSON_KEY, submissionsValidator); - this.autoSubmit.restoreState(verifyOptionalObjectProperty(x, AUTOSUBMIT_JSON_KEY, verifyBoolean)); + this.autoSubmit.restoreState( + verifyOptionalObjectProperty(x, AUTOSUBMIT_JSON_KEY, verifyBoolean)); + } +} + + +class FindPathState extends RefCounted implements Trackable { + changed = new NullarySignal(); + triggerPathUpdate = new NullarySignal(); + + source = new TrackableValue(undefined, x => x); + target = new TrackableValue(undefined, x => x); + centroids = new TrackableValue([], x => x); + precisionMode = new TrackableBoolean(true); + + get path() { + const path: Line[] = []; + const {source: {value: source}, target: {value: target}, centroids: {value: centroids}} = this; + if (!source || !target || centroids.length === 0) { + return path; + } + for (let i = 0; i < centroids.length - 1; i++) { + const pointA = centroids[i]; + const pointB = centroids[i + 1]; + const line: Line = { + pointA: vec3.fromValues(pointA[0], pointA[1], pointA[2]), + pointB: vec3.fromValues(pointB[0], pointB[1], pointB[2]), + id: '', + type: AnnotationType.LINE, + properties: [], + }; + path.push(line); + } + const firstLine: Line = { + pointA: source.position, + pointB: path[0].pointA, + id: '', + type: AnnotationType.LINE, + properties: [], + }; + const lastLine: Line = { + pointA: path[path.length - 1].pointB, + pointB: target.position, + id: '', + type: AnnotationType.LINE, + properties: [], + }; + + return [firstLine, ...path, lastLine]; + } + + constructor() { + super(); + this.registerDisposer(this.source.changed.add(() => { + this.centroids.reset(); + this.changed.dispatch(); + })); + this.registerDisposer(this.target.changed.add(() => { + this.centroids.reset(); + this.changed.dispatch(); + })); + this.registerDisposer(this.centroids.changed.add(this.changed.dispatch)); + } + + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + const {source: {value: source}, target: {value: target}} = this; + const newValue = newValues.size === 1 ? [...newValues][0] : undefined; + const sourceChanged = !!source && oldValues.has(source.rootId); + const targetChanged = !!target && oldValues.has(target.rootId); + if (newValue) { + if (sourceChanged) { + source.rootId = newValue; + } + if (targetChanged) { + target.rootId = newValue; + } + // don't want to fire off multiple changed + if (sourceChanged || targetChanged) { + if (this.centroids.value.length) { + this.centroids.reset(); + this.triggerPathUpdate.dispatch(); + } else { + this.changed.dispatch(); + } + } + } else { + if (sourceChanged || targetChanged) { + this.reset(); + } + } + } + + reset() { + this.source.reset(); + this.target.reset(); + this.centroids.reset(); + this.precisionMode.reset(); + } + + toJSON() { + const { + source: {value: source}, + target: {value: target}, + centroids, + precisionMode, + } = this; + return { + [SOURCE_JSON_KEY]: source ? segmentSelectionToJSON(source) : undefined, + [TARGET_JSON_KEY]: target ? segmentSelectionToJSON(target) : undefined, + [CENTROIDS_JSON_KEY]: centroids.toJSON(), + [PRECISION_MODE_JSON_KEY]: precisionMode.toJSON(), + }; + } + + restoreState(x: any) { + verifyOptionalObjectProperty(x, SOURCE_JSON_KEY, value => { + this.source.restoreState(restoreSegmentSelection(value)); + }); + verifyOptionalObjectProperty(x, TARGET_JSON_KEY, value => { + this.target.restoreState(restoreSegmentSelection(value)); + }); + verifyOptionalObjectProperty(x, CENTROIDS_JSON_KEY, value => { + this.centroids.restoreState(value); + }); + verifyOptionalObjectProperty(x, PRECISION_MODE_JSON_KEY, value => { + this.precisionMode.restoreState(value); + }); } } @@ -676,8 +854,28 @@ class MulticutState extends RefCounted implements Trackable { this.registerDisposer(this.sources.changed.add(this.changed.dispatch)); } + replaceSegments(oldValues: Uint64Set, newValues: Uint64Set) { + const newValue = newValues.size === 1 ? [...newValues][0] : undefined; + const {focusSegment: {value: focusSegment}} = this; + + if (focusSegment && oldValues.has(focusSegment)) { + if (newValue) { + this.focusSegment.value = newValue; + for (const sink of this.sinks) { + sink.rootId = newValue; + } + for (const source of this.sources) { + source.rootId = newValue; + } + this.changed.dispatch(); + } else { + this.reset(); + } + } + } + reset() { - this.focusSegment.value = undefined; + this.focusSegment.reset(); this.blueGroup.value = false; this.sinks.clear(); this.sources.clear(); @@ -685,14 +883,6 @@ class MulticutState extends RefCounted implements Trackable { toJSON() { const {focusSegment, sinks, sources} = this; - - const segmentSelectionToJSON = (x: SegmentSelection) => { - return { - [SEGMENT_ID_JSON_KEY]: x.segmentId.toJSON(), [ROOT_ID_JSON_KEY]: x.rootId.toJSON(), - [POSITION_JSON_KEY]: [...x.position], - } - }; - return { [FOCUS_SEGMENT_JSON_KEY]: focusSegment.toJSON(), [SINKS_JSON_KEY]: [...sinks].map(segmentSelectionToJSON), @@ -749,39 +939,43 @@ class MulticutState extends RefCounted implements Trackable { class GraphConnection extends SegmentationGraphSourceConnection { public annotationLayerStates: AnnotationLayerState[] = []; public mergeAnnotationState: AnnotationLayerState; + public findPathAnnotationState: AnnotationLayerState; constructor( public graph: GrapheneGraphSource, private layer: SegmentationUserLayer, private chunkSource: GrapheneMultiscaleVolumeChunkSource, public state: GrapheneState) { super(graph, layer.displayState.segmentationGroupState.value); const segmentsState = layer.displayState.segmentationGroupState.value; - segmentsState.selectedSegments.changed.add((segmentIds: Uint64[]|Uint64|null, add: boolean) => { - if (segmentIds !== null) { - segmentIds = Array().concat(segmentIds); - } - this.selectedSegmentsChanged(segmentIds, add); - }); + this.registerDisposer(segmentsState.selectedSegments.changed.add( + (segmentIds: Uint64[]|Uint64|null, add: boolean) => { + if (segmentIds !== null) { + segmentIds = Array().concat(segmentIds); + } + this.selectedSegmentsChanged(segmentIds, add); + })); - segmentsState.visibleSegments.changed.add((segmentIds: Uint64[]|Uint64|null, add: boolean) => { - if (segmentIds !== null) { - segmentIds = Array().concat(segmentIds); - } - this.visibleSegmentsChanged(segmentIds, add); - }); + this.registerDisposer(segmentsState.visibleSegments.changed.add( + (segmentIds: Uint64[]|Uint64|null, add: boolean) => { + if (segmentIds !== null) { + segmentIds = Array().concat(segmentIds); + } + this.visibleSegmentsChanged(segmentIds, add); + })); - const {annotationLayerStates, state: {multicutState}} = this; + const {annotationLayerStates, state: {multicutState, findPathState}} = this; const loadedSubsource = getGraphLoadedSubsource(layer)!; const redGroup = makeColoredAnnotationState(layer, loadedSubsource, 'sinks', RED_COLOR); const blueGroup = makeColoredAnnotationState(layer, loadedSubsource, 'sources', BLUE_COLOR); synchronizeAnnotationSource(multicutState.sinks, redGroup); - synchronizeAnnotationSource(multicutState.sources, blueGroup) + synchronizeAnnotationSource(multicutState.sources, blueGroup); annotationLayerStates.push(redGroup, blueGroup); if (layer.tool.value instanceof MergeSegmentsPlaceLineTool) { layer.tool.value = undefined; } - this.mergeAnnotationState = makeColoredAnnotationState(layer, loadedSubsource, "grapheneMerge", RED_COLOR); + this.mergeAnnotationState = + makeColoredAnnotationState(layer, loadedSubsource, 'grapheneMerge', RED_COLOR); { const {mergeState} = state; @@ -796,7 +990,7 @@ class GraphConnection extends SegmentationGraphSourceConnection { } // initialize source changes - mergeAnnotationState.source.childAdded.add((x) => { + this.registerDisposer(mergeAnnotationState.source.childAdded.add((x) => { const annotation = x as Line; const relatedSegments = annotation.relatedSegments![0]; const visibles = relatedSegments.map(x => visibleSegments.has(x)); @@ -817,13 +1011,14 @@ class GraphConnection extends SegmentationGraphSourceConnection { tool.value.deactivate(); } }, 0); - StatusMessage.showTemporaryMessage(`Maximum of ${MAX_MERGE_COUNT} simultanous merges allowed.`); + StatusMessage.showTemporaryMessage( + `Maximum of ${MAX_MERGE_COUNT} simultanous merges allowed.`); } - }); + })); - mergeAnnotationState.source.childCommitted.add((x) => { + this.registerDisposer(mergeAnnotationState.source.childCommitted.add((x) => { const ref = mergeAnnotationState.source.getReference(x); - const annotation = ref.value as Line|undefined; + const annotation = ref.value as Line | undefined; if (annotation) { const relatedSegments = annotation.relatedSegments![0]; const visibles = relatedSegments.map(x => visibleSegments.has(x)); @@ -836,7 +1031,7 @@ class GraphConnection extends SegmentationGraphSourceConnection { StatusMessage.showTemporaryMessage(`Cannot merge a hidden segment.`); } const existingSubmission = merges.value.find(x => x.id === ref.id); - if (existingSubmission && !existingSubmission?.locked) { // how would it be locked? + if (existingSubmission && !existingSubmission?.locked) { // how would it be locked? const newSubmission = lineToSubmission(annotation, false); existingSubmission.sink = newSubmission.sink; existingSubmission.source = newSubmission.source; @@ -847,9 +1042,9 @@ class GraphConnection extends SegmentationGraphSourceConnection { } } ref.dispose(); - }); + })); - mergeAnnotationState.source.childDeleted.add((id) => { + this.registerDisposer(mergeAnnotationState.source.childDeleted.add((id) => { let changed = false; const filtered = merges.value.filter(x => { const keep = x.id !== id || x.locked; @@ -861,8 +1056,52 @@ class GraphConnection extends SegmentationGraphSourceConnection { if (changed) { merges.value = filtered; } - }); + })); } + + const findPathGroup = + makeColoredAnnotationState(layer, loadedSubsource, 'findpath', WHITE_COLOR); + this.findPathAnnotationState = findPathGroup; + findPathGroup.source.childDeleted.add(annotationId => { + if (findPathState.source.value?.annotationReference?.id === annotationId) { + findPathState.source.value = undefined; + } + if (findPathState.target.value?.annotationReference?.id === annotationId) { + findPathState.target.value = undefined; + } + }); + const findPathChanged = () => { + const {path, source, target} = findPathState; + const annotationSource = findPathGroup.source; + if (source.value && !source.value.annotationReference) { + addSelection(annotationSource, source.value, 'find path source'); + } + if (target.value && !target.value.annotationReference) { + addSelection(annotationSource, target.value, 'find path target'); + } + for (const annotation of annotationSource) { + if (annotation.id !== source.value?.annotationReference?.id && + annotation.id !== target.value?.annotationReference?.id) { + annotationSource.delete(annotationSource.getReference(annotation.id)); + } + } + for (const line of path) { + // line.id = ''; // TODO, is it a bug that this is necessary? annotationMap is empty if I + // step through it but logging shows it isn't empty + annotationSource.add(line); + } + }; + this.registerDisposer(findPathState.changed.add(findPathChanged)); + this.registerDisposer(findPathState.triggerPathUpdate.add(() => { + const loadedSubsource = getGraphLoadedSubsource(this.layer)!; + const annotationToNanometers = + loadedSubsource.loadedDataSource.transform.inputSpace.value.scales.map(x => x / 1e-9); + this.submitFindPath(findPathState.precisionMode.value, annotationToNanometers) + .then(success => { + success; + }); + })); + findPathChanged(); // initial state } createRenderLayers( @@ -903,14 +1142,14 @@ class GraphConnection extends SegmentationGraphSourceConnection { } for (const segmentId of segments) { if (!added) { - const segmentCount = [...segmentsState.segmentEquivalences.setElements(segmentId)].length; // Approximation + const segmentCount = + [...segmentsState.segmentEquivalences.setElements(segmentId)].length; // Approximation segmentsState.segmentEquivalences.deleteSet(segmentId); if (this.lastDeselectionMessage && this.lastDeselectionMessageExists) { this.lastDeselectionMessage.dispose(); this.lastDeselectionMessageExists = false; } - this.lastDeselectionMessage = - StatusMessage.showMessage(`Hid ${segmentCount} segments.`); + this.lastDeselectionMessage = StatusMessage.showMessage(`Hid ${segmentCount} segments.`); this.lastDeselectionMessageExists = true; setTimeout(() => { if (this.lastDeselectionMessageExists) { @@ -976,8 +1215,8 @@ class GraphConnection extends SegmentationGraphSourceConnection { if (meshSource) { for (const segment of segments) { meshSource.rpc!.invoke( - GRAPHENE_MESH_NEW_SEGMENT_RPC_ID, - {'rpcId': meshSource.rpcId!, 'segment': segment.toString()}); + GRAPHENE_MESH_NEW_SEGMENT_RPC_ID, + {'rpcId': meshSource.rpcId!, 'segment': segment.toString()}); } } } @@ -991,7 +1230,8 @@ class GraphConnection extends SegmentationGraphSourceConnection { return false; } else { const splitRoots = await this.graph.graphServer.splitSegments( - [...sinks], [...sources], annotationToNanometers); + [...sinks].map(x => selectionInNanometers(x, annotationToNanometers)), + [...sources].map(x => selectionInNanometers(x, annotationToNanometers))); if (splitRoots.length === 0) { StatusMessage.showTemporaryMessage(`No split found.`, 3000); return false; @@ -1007,114 +1247,120 @@ class GraphConnection extends SegmentationGraphSourceConnection { this.meshAddNewSegments(splitRoots); segmentsState.selectedSegments.add(splitRoots); segmentsState.visibleSegments.add(splitRoots); + const oldValues = new Uint64Set(); + oldValues.add(focusSegment); + const newValues = new Uint64Set(); + newValues.add(splitRoots); + this.state.replaceSegments(oldValues, newValues); return true; } } } - deleteMergeSubmission = (submission: MergeSubmission) => { - const {mergeAnnotationState} = this; - submission.locked = false; - mergeAnnotationState.source.delete(mergeAnnotationState.source.getReference(submission.id)); - } + deleteMergeSubmission = + (submission: MergeSubmission) => { + const {mergeAnnotationState} = this; + submission.locked = false; + mergeAnnotationState.source.delete(mergeAnnotationState.source.getReference(submission.id)); + } - private submitMerge = async (submission: MergeSubmission, attempts=1): Promise => { - this.graph - const loadedSubsource = getGraphLoadedSubsource(this.layer)!; - const annotationToNanometers = loadedSubsource.loadedDataSource.transform.inputSpace.value.scales.map(x => x / 1e-9); - submission.error = undefined; - for (let i = 1; i <= attempts; i++) { - try { - return await this.graph.graphServer.mergeSegments(submission.sink, submission.source!, annotationToNanometers); - } catch (err) { - if (i === attempts) { - submission.error = err.message || "unknown"; - throw err; + private submitMerge = async(submission: MergeSubmission, attempts = 1): + Promise => { + const loadedSubsource = getGraphLoadedSubsource(this.layer)!; + const annotationToNanometers = + loadedSubsource.loadedDataSource.transform.inputSpace.value.scales.map(x => x / 1e-9); + submission.error = undefined; + for (let i = 1; i <= attempts; i++) { + try { + const newRoot = await this.graph.graphServer.mergeSegments( + selectionInNanometers(submission.sink, annotationToNanometers), + selectionInNanometers(submission.source!, annotationToNanometers)); + const oldValues = new Uint64Set(); + oldValues.add(submission.sink.rootId); + oldValues.add(submission.source!.rootId); + const newValues = new Uint64Set(); + newValues.add(newRoot); + this.state.replaceSegments(oldValues, newValues); + return newRoot; + } catch (err) { + if (i === attempts) { + submission.error = err.message || 'unknown'; + throw err; + } + } } - } - } - return Uint64.ZERO; // appease typescript - } + return Uint64.ZERO; // appease typescript + } async bulkMerge(submissions: MergeSubmission[]) { - const {merges} = this.state.mergeState; - const bulkMergeHelper = (submissions: MergeSubmission[]): Promise => { - return new Promise(f => { - if (submissions.length === 0) { - f([]); - return; - } - const segmentsToRemove: Uint64[] = []; - const replaceSegment = (a: Uint64, b: Uint64) => { - segmentsToRemove.push(a); - for (const submission of submissions) { - if (submission.source && Uint64.equal(submission.source.rootId, a)) { - submission.source.rootId = b; - } - if (Uint64.equal(submission.sink.rootId, a)) { - submission.sink.rootId = b; - } + const {merges} = this.state.mergeState; + const bulkMergeHelper = (submissions: MergeSubmission[]): Promise => { + return new Promise(f => { + if (submissions.length === 0) { + f([]); + return; + } + const segmentsToRemove: Uint64[] = []; + let completed = 0; + let activeLoops = 0; + const loop = (completedAt: number, pending: MergeSubmission[]) => { + if (completed === submissions.length || pending.length === 0) return; + activeLoops++; + let failed: MergeSubmission[] = []; + const checkDone = () => { + loopDone++; + if (loopDone === pending.length) { + activeLoops -= 1; + } + if (activeLoops === 0) { + f(segmentsToRemove); } }; - let completed = 0; - let activeLoops = 0; - const loop = (completedAt: number, pending: MergeSubmission[]) => { - if (completed === submissions.length || pending.length === 0) return; - activeLoops++; - let failed: MergeSubmission[] = []; - const checkDone = () => { - loopDone++; - if (loopDone === pending.length) { - activeLoops -= 1; - } - if (activeLoops === 0) { - f(segmentsToRemove); - } - }; - let loopDone = 0; - for (const submission of pending) { - submission.locked = true; - submission.status = 'trying...'; - merges.changed.dispatch(); - this.submitMerge(submission, 3).then(mergedRoot => { - replaceSegment(submission.source!.rootId, mergedRoot); - replaceSegment(submission.sink.rootId, mergedRoot); - submission.status = 'done'; - submission.mergedRoot = mergedRoot; - merges.changed.dispatch(); - completed += 1; - loop(completed, failed); - failed = []; - checkDone(); - wait(5000).then(() => { - this.deleteMergeSubmission(submission); - }); - }).catch(() => { - merges.changed.dispatch(); - failed.push(submission); - if (completed > completedAt) { + let loopDone = 0; + for (const submission of pending) { + submission.locked = true; + submission.status = 'trying...'; + merges.changed.dispatch(); + const segments = [submission.source!.rootId, submission.sink.rootId]; + this.submitMerge(submission, 3) + .then(mergedRoot => { + segmentsToRemove.push(...segments); + submission.status = 'done'; + submission.mergedRoot = mergedRoot; + merges.changed.dispatch(); + completed += 1; loop(completed, failed); failed = []; - } - checkDone(); - }); - } - }; - loop(completed, submissions); - }); - }; + checkDone(); + wait(5000).then(() => { + this.deleteMergeSubmission(submission); + }); + }) + .catch(() => { + merges.changed.dispatch(); + failed.push(submission); + if (completed > completedAt) { + loop(completed, failed); + failed = []; + } + checkDone(); + }); + } + }; + loop(completed, submissions); + }); + }; - submissions = submissions.filter(x => !x.locked && x.source); - const segmentsToRemove = await bulkMergeHelper(submissions); - const segmentsToAdd: Uint64[] = []; - for (const submission of submissions) { - if (submission.error) { - submission.locked = false; - submission.status = submission.error; - } else if (submission.mergedRoot) { - segmentsToAdd.push(submission.mergedRoot); - } + submissions = submissions.filter(x => !x.locked && x.source); + const segmentsToRemove = await bulkMergeHelper(submissions); + const segmentsToAdd: Uint64[] = []; + for (const submission of submissions) { + if (submission.error) { + submission.locked = false; + submission.status = submission.error; + } else if (submission.mergedRoot) { + segmentsToAdd.push(submission.mergedRoot); } const segmentsState = this.layer.displayState.segmentationGroupState.value; const {visibleSegments, selectedSegments} = segmentsState; @@ -1125,6 +1371,26 @@ class GraphConnection extends SegmentationGraphSourceConnection { visibleSegments.add(latestRoots); merges.changed.dispatch(); } + const segmentsState = this.layer.displayState.segmentationGroupState.value; + const {visibleSegments, selectedSegments} = segmentsState; + selectedSegments.delete(segmentsToRemove); + const latestRoots = await this.graph.graphServer.filterLatestRoots(segmentsToAdd); + selectedSegments.add(latestRoots); + visibleSegments.add(latestRoots); + merges.changed.dispatch(); + } + + async submitFindPath(precisionMode: boolean, annotationToNanometers: Float64Array): + Promise { + const {state: {findPathState}} = this; + const {source, target} = findPathState; + if (!source.value || !target.value) return false; + const centroids = await this.graph.findPath( + source.value, target.value, precisionMode, annotationToNanometers); + StatusMessage.showTemporaryMessage('Path found!', 5000); + findPathState.centroids.value = centroids; + return true; + } } async function parseGrapheneError(e: HttpError) { @@ -1141,37 +1407,44 @@ async function parseGrapheneError(e: HttpError) { } async function withErrorMessageHTTP(promise: Promise, options: { - initialMessage?: string, - errorPrefix: string, - }): Promise { - let status: StatusMessage|undefined = undefined; - let dispose = () => {}; - if (options.initialMessage) { - status = new StatusMessage(true); - status.setText(options.initialMessage); - dispose = status.dispose.bind(status); - } - try { - const response = await promise; - dispose(); - return response; - } catch (e) { - if (e instanceof HttpError && e.response) { - const {errorPrefix = ''} = options; - const msg = await parseGrapheneError(e); - if (msg) { - if (!status) { - status = new StatusMessage(true); - } - status.setErrorMessage(errorPrefix + msg); - status.setVisible(true); - throw new Error(`[${e.response.status}] ${errorPrefix}${msg}`); - } + initialMessage?: string, errorPrefix: string, +}): Promise { + let status: StatusMessage|undefined = undefined; + let dispose = () => {}; + if (options.initialMessage) { + status = new StatusMessage(true); + status.setText(options.initialMessage); + dispose = status.dispose.bind(status); + } + try { + const response = await promise; + dispose(); + return response; + } catch (e) { + if (e instanceof HttpError && e.response) { + const {errorPrefix = ''} = options; + const msg = await parseGrapheneError(e) || 'unknown error'; + if (!status) { + status = new StatusMessage(true); } - throw e; + status.setErrorMessage(errorPrefix + msg); + status.setVisible(true); + throw new Error(`[${e.response.status}] ${errorPrefix}${msg}`); } + throw e; + } } +const selectionInNanometers = + (selection: SegmentSelection, annotationToNanometers: Float64Array): SegmentSelection => { + const {rootId, segmentId, position} = selection; + return { + rootId, + segmentId, + position: position.map((val, i) => val * annotationToNanometers[i]) + }; + }; + export const GRAPH_SERVER_NOT_SPECIFIED = Symbol('Graph Server Not Specified.'); class GrapheneGraphServerInterface { @@ -1194,9 +1467,7 @@ class GrapheneGraphServerInterface { return Uint64.parseString(jsonResp['root_id']); } - async mergeSegments( - first: SegmentSelection, second: SegmentSelection, - annotationToNanometers: Float64Array): Promise { + async mergeSegments(first: SegmentSelection, second: SegmentSelection): Promise { const {url} = this; if (url === '') { return Promise.reject(GRAPH_SERVER_NOT_SPECIFIED); @@ -1206,14 +1477,8 @@ class GrapheneGraphServerInterface { this.credentialsProvider, `${url}/merge?int64_as_str=1`, { method: 'POST', body: JSON.stringify([ - [ - String(first.segmentId), - ...first.position.map((val, i) => val * annotationToNanometers[i]) - ], - [ - String(second.segmentId), - ...second.position.map((val, i) => val * annotationToNanometers[i]) - ] + [String(first.segmentId), ...first.position], + [String(second.segmentId), ...second.position] ]) }, responseIdentity); @@ -1231,30 +1496,21 @@ class GrapheneGraphServerInterface { } } - async splitSegments( - first: SegmentSelection[], second: SegmentSelection[], - annotationToNanometers: Float64Array): Promise { + async splitSegments(first: SegmentSelection[], second: SegmentSelection[]): Promise { const {url} = this; if (url === '') { return Promise.reject(GRAPH_SERVER_NOT_SPECIFIED); } - const promise = - cancellableFetchSpecialOk( - this.credentialsProvider, `${url}/split?int64_as_str=1`, { - method: 'POST', - body: JSON.stringify({ - 'sources': first.map( - x => - [String(x.segmentId), - ...x.position.map((val, i) => val * annotationToNanometers[i])]), - 'sinks': second.map( - x => - [String(x.segmentId), - ...x.position.map((val, i) => val * annotationToNanometers[i])]) - }) - }, - responseIdentity); + const promise = cancellableFetchSpecialOk( + this.credentialsProvider, `${url}/split?int64_as_str=1`, { + method: 'POST', + body: JSON.stringify({ + 'sources': first.map(x => [String(x.segmentId), ...x.position]), + 'sinks': second.map(x => [String(x.segmentId), ...x.position]), + }) + }, + responseIdentity); const response = await withErrorMessageHTTP(promise, { initialMessage: `Splitting ${first.length} sources from ${second.length} sinks`, @@ -1271,16 +1527,14 @@ class GrapheneGraphServerInterface { async filterLatestRoots(segments: Uint64[]): Promise { const url = `${this.url}/is_latest_roots`; - const promise = cancellableFetchSpecialOk(this.credentialsProvider, url, { - method: 'POST', - body: JSON.stringify({ - "node_ids": segments.map(x => x.toJSON()) - }), - }, responseIdentity); + const promise = cancellableFetchSpecialOk( + this.credentialsProvider, url, { + method: 'POST', + body: JSON.stringify({'node_ids': segments.map(x => x.toJSON())}), + }, + responseIdentity); - const response = await withErrorMessageHTTP(promise, { - errorPrefix: `Could not check latest: ` - }); + const response = await withErrorMessageHTTP(promise, {errorPrefix: `Could not check latest: `}); const jsonResp = await response.json(); const res: Uint64[] = []; @@ -1291,15 +1545,53 @@ class GrapheneGraphServerInterface { } return res; } + + async findPath(first: SegmentSelection, second: SegmentSelection, precisionMode: boolean) { + const {url} = this; + if (url === '') { + return Promise.reject(GRAPH_SERVER_NOT_SPECIFIED); + } + const promise = cancellableFetchSpecialOk( + this.credentialsProvider, + `${url}/graph/find_path?int64_as_str=1&precision_mode=${Number(precisionMode)}`, { + method: 'POST', + body: JSON.stringify([ + [String(first.rootId), ...first.position], + [String(second.rootId), ...second.position], + ]), + }, + responseIdentity); + + const response = await withErrorMessageHTTP(promise, { + initialMessage: `Finding path between ${first.segmentId} and ${second.segmentId}`, + errorPrefix: 'Path finding failed: ' + }); + const jsonResponse = await response.json(); + const supervoxelCentroidsKey = 'centroids_list'; + let centroids = verifyObjectProperty( + jsonResponse, supervoxelCentroidsKey, x => parseArray(x, verifyFloatArray)); + const missingL2IdsKey = 'failed_l2_ids'; + const missingL2Ids = jsonResponse[missingL2IdsKey]; + if (missingL2Ids && missingL2Ids.length > 0) { + StatusMessage.showTemporaryMessage( + 'Some level 2 meshes are missing, so the path shown may have a poor level of detail.'); + } + const l2_path = verifyOptionalObjectProperty(jsonResponse, 'l2_path', verifyStringArray); + return { + centroids, + l2_path, + }; + } } class GrapheneGraphSource extends SegmentationGraphSource { private connections = new Set(); public graphServer: GrapheneGraphServerInterface; + private l2CacheAvailable: boolean|undefined = undefined; constructor( public info: GrapheneMultiscaleVolumeInfo, - credentialsProvider: SpecialProtocolCredentialsProvider, + private credentialsProvider: SpecialProtocolCredentialsProvider, private chunkSource: GrapheneMultiscaleVolumeChunkSource, public state: GrapheneState) { super(); this.graphServer = @@ -1322,10 +1614,69 @@ class GrapheneGraphSource extends SegmentationGraphSource { VisibleSegmentEquivalencePolicy.NONREPRESENTATIVE_EXCLUDED; } + async isL2CacheUrlAvailable() { + if (this.l2CacheAvailable !== undefined) { + return this.l2CacheAvailable; + } + try { + const {l2CacheUrl, table} = this.info.app; + const tableMapping = await cancellableFetchSpecialOk( + undefined, `${l2CacheUrl}/table_mapping`, {}, responseJson); + this.l2CacheAvailable = !!(tableMapping && tableMapping[table]); + return this.l2CacheAvailable; + } catch (e) { + console.error('e', e); + return false; + } + } + getRoot(segment: Uint64) { return this.graphServer.getRoot(segment); } + async findPath( + first: SegmentSelection, second: SegmentSelection, precisionMode: boolean, + annotationToNanometers: Float64Array): Promise { + const {l2CacheUrl, table} = this.info.app; + const l2CacheAvailable = precisionMode && + await this.isL2CacheUrlAvailable(); // don't check if available if we don't need it + let {centroids, l2_path} = await this.graphServer.findPath( + selectionInNanometers(first, annotationToNanometers), + selectionInNanometers(second, annotationToNanometers), precisionMode && !l2CacheAvailable); + if (precisionMode && l2CacheAvailable && l2_path) { + const repCoordinatesUrl = `${l2CacheUrl}/table/${table}/attributes`; + try { + const res = await cancellableFetchSpecialOk( + this.credentialsProvider, repCoordinatesUrl, { + method: 'POST', + body: JSON.stringify({ + l2_ids: l2_path, + }), + }, + responseJson); + + // many reasons why an l2 id might not have info + // l2 cache has a process that takes time for new ids (even hours) + // maybe a small fraction have no info + // sometime l2 is so small (single voxel), it is ignored by l2 + // best to just drop those points + centroids = l2_path + .map(id => { + return verifyOptionalObjectProperty(res, id, x => { + return verifyIntegerArray(x['rep_coord_nm']); + }); + }) + .filter((x): x is number[] => x !== undefined); + } catch (e) { + console.log('e', e); + } + } + const centroidsTransformed = centroids.map((point: number[]) => { + return point.map((val, i) => val / annotationToNanometers[i]); + }); + return centroidsTransformed; + } + tabContents( layer: SegmentationUserLayer, context: DependentViewContext, tab: SegmentationGraphSourceTab) { @@ -1341,6 +1692,9 @@ class GrapheneGraphSource extends SegmentationGraphSource { toolbox.appendChild(makeToolButton( context, layer.toolBinder, {toolJson: GRAPHENE_MERGE_SEGMENTS_TOOL_ID, label: 'Merge', title: 'Merge segments'})); + toolbox.appendChild(makeToolButton( + context, layer.toolBinder, + {toolJson: GRAPHENE_FIND_PATH_TOOL_ID, label: 'Find Path', title: 'Find Path'})); parent.appendChild(toolbox); parent.appendChild( context @@ -1484,6 +1838,7 @@ class SliceViewPanelChunkedGraphLayer extends SliceViewPanelRenderLayer { const GRAPHENE_MULTICUT_SEGMENTS_TOOL_ID = 'grapheneMulticutSegments'; const GRAPHENE_MERGE_SEGMENTS_TOOL_ID = 'grapheneMergeSegments'; +const GRAPHENE_FIND_PATH_TOOL_ID = 'grapheneFindPath'; class MulticutAnnotationLayerView extends AnnotationLayerView { private _annotationStates: MergedAnnotationStates; @@ -1507,60 +1862,59 @@ class MulticutAnnotationLayerView extends AnnotationLayerView { } } -const synchronizeAnnotationSource = (source: WatchableSet, state: AnnotationLayerState) => { - const annotationSource = state.source; - - annotationSource.childDeleted.add(annotationId => { - const selection = [...source].find(selection => selection.annotationReference?.id === annotationId) - if (selection) source.delete(selection); - }); - - const addSelection = (selection: SegmentSelection) => { - const annotation: Point = { - id: '', - point: selection.position, - type: AnnotationType.POINT, - properties: [], - relatedSegments: [[selection.segmentId, selection.rootId]], - }; - const ref = annotationSource.add(annotation); - selection.annotationReference = ref; - } - - source.changed.add((x, add) => { - if (x === null) { - for (const annotation of annotationSource) { - // using .clear does not remove annotations from the list - // (this.blueGroupAnnotationState.source as LocalAnnotationSource).clear(); - annotationSource.delete(annotationSource.getReference(annotation.id)); - } - return; - } - - if (add) { - addSelection(x); - } else if (x.annotationReference) { - annotationSource.delete(x.annotationReference); +const addSelection = + (source: AnnotationSource|MultiscaleAnnotationSource, selection: SegmentSelection, + description?: string) => { + const annotation: Point = { + id: '', + point: selection.position, + type: AnnotationType.POINT, + properties: [], + relatedSegments: [[selection.segmentId, selection.rootId]], + description, + }; + const ref = source.add(annotation); + selection.annotationReference = ref; } - }); - // load initial state - for (const selection of source) { - addSelection(selection); - } -} +const synchronizeAnnotationSource = + (source: WatchableSet, state: AnnotationLayerState) => { + const annotationSource = state.source; + annotationSource.childDeleted.add(annotationId => { + const selection = + [...source].find(selection => selection.annotationReference?.id === annotationId) + if (selection) source.delete(selection); + }); + source.changed.add((x, add) => { + if (x === null) { + for (const annotation of annotationSource) { + annotationSource.delete(annotationSource.getReference(annotation.id)); + } + return; + } + if (add) { + addSelection(annotationSource, x); + } else if (x.annotationReference) { + annotationSource.delete(x.annotationReference); + } + }); + // load initial state + for (const selection of source) { + addSelection(annotationSource, selection); + } + }; function getMousePositionInLayerCoordinates( - unsnappedPosition: Float32Array, layer: SegmentationUserLayer): Float32Array| - undefined { + unsnappedPosition: Float32Array, layer: SegmentationUserLayer): Float32Array|undefined { const loadedSubsource = getGraphLoadedSubsource(layer)!; const modelTransform = loadedSubsource.getRenderLayerTransform(); - const chunkTransform = makeValueOrError(() => getChunkTransformParameters(valueOrThrow(modelTransform.value))); + const chunkTransform = + makeValueOrError(() => getChunkTransformParameters(valueOrThrow(modelTransform.value))); if (chunkTransform.error !== undefined) return undefined; const chunkPosition = new Float32Array(chunkTransform.modelTransform.unpaddedRank); if (!getChunkPositionFromCombinedGlobalLocalPositions( - chunkPosition, unsnappedPosition, layer.localPosition.value, - chunkTransform.layerRank, chunkTransform.combinedGlobalLocalToChunkTransform)) { + chunkPosition, unsnappedPosition, layer.localPosition.value, chunkTransform.layerRank, + chunkTransform.combinedGlobalLocalToChunkTransform)) { return undefined; } return chunkPosition; @@ -1571,7 +1925,7 @@ const getPoint = (layer: SegmentationUserLayer, mouseState: MouseSelectionState) return getMousePositionInLayerCoordinates(mouseState.unsnappedPosition, layer); } return undefined; -} +}; const MULTICUT_SEGMENTS_INPUT_EVENT_MAP = EventActionMap.fromObject({ 'at:shift?+control+mousedown0': {action: 'set-anchor'}, @@ -1593,7 +1947,7 @@ class MulticutSegmentsTool extends LayerTool { const {body, header} = makeToolActivationStatusMessageWithHeader(activation); header.textContent = 'Multicut segments'; - body.classList.add('graphene-multicut-status'); + body.classList.add('graphene-tool-status', 'graphene-multicut'); body.appendChild(makeIcon({ text: 'Swap', title: 'Swap group', @@ -1619,7 +1973,7 @@ class MulticutSegmentsTool extends LayerTool { activation.cancel(); } }); - } + }; const submitIcon = makeIcon({ text: 'Submit', title: 'Submit multicut', @@ -1768,11 +2122,12 @@ const maybeGetSelection = }; }; -const wait = (t: number) => { - return new Promise((f, _r) => { - setTimeout(f, t); - }); -} +const wait = + (t: number) => { + return new Promise((f, _r) => { + setTimeout(f, t); + }); + } interface MergeSubmission { id: string; @@ -1790,7 +2145,8 @@ export class MergeSegmentsPlaceLineTool extends PlaceLineTool { super(layer, {}); const {inProgressAnnotation} = this; const {displayState} = annotationState; - if (!displayState) return; // TODO, this happens when reloading the page when a toggle tool is up + if (!displayState) + return; // TODO, this happens when reloading the page when a toggle tool is up const {disablePicking} = displayState; this.registerDisposer(inProgressAnnotation.changed.add(() => { disablePicking.value = inProgressAnnotation.value !== undefined; @@ -1822,7 +2178,7 @@ function lineToSubmission(line: Line, pending: boolean): MergeSubmission { res.source = { position: line.pointB.slice(), rootId: relatedSegments[2].clone(), - segmentId:relatedSegments[3].clone(), + segmentId: relatedSegments[3].clone(), }; } return res; @@ -1835,7 +2191,9 @@ function mergeToLine(submission: MergeSubmission): Line { type: AnnotationType.LINE, pointA: sink.position.slice(), pointB: source!.position.slice(), - relatedSegments: [[sink.rootId.clone(), sink.segmentId.clone(), source!.rootId.clone(), source!.segmentId.clone()]], + relatedSegments: [[ + sink.rootId.clone(), sink.segmentId.clone(), source!.rootId.clone(), source!.segmentId.clone() + ]], properties: [], }; return res; @@ -1843,8 +2201,6 @@ function mergeToLine(submission: MergeSubmission): Line { const MAX_MERGE_COUNT = 10; -// on error, copy (also clean up error message) - const MERGE_SEGMENTS_INPUT_EVENT_MAP = EventActionMap.fromObject({ 'at:shift?+enter': {action: 'submit'}, }); @@ -1857,27 +2213,29 @@ class MergeSegmentsTool extends LayerTool { if (mergeState === undefined) return; const {merges, autoSubmit} = mergeState; - const lineTool = new MergeSegmentsPlaceLineTool(this.layer, graphConnection.mergeAnnotationState); + const lineTool = + new MergeSegmentsPlaceLineTool(this.layer, graphConnection.mergeAnnotationState); tool.value = lineTool; activation.registerDisposer(() => { tool.value = undefined; }); const {body, header} = makeToolActivationStatusMessageWithHeader(activation); header.textContent = 'Merge segments'; - body.classList.add('graphene-merge-segments-status'); + body.classList.add('graphene-tool-status', 'graphene-merge-segments'); activation.bindInputEventMap(MERGE_SEGMENTS_INPUT_EVENT_MAP); const submitAction = async () => { if (merges.value.filter(x => x.locked).length) return; submitIcon.classList.toggle('disabled', true); await graphConnection.bulkMerge(merges.value); submitIcon.classList.toggle('disabled', false); - } + }; const submitIcon = makeIcon({ text: 'Submit', title: 'Submit merge', onClick: async () => { submitAction(); - }}); + } + }); body.appendChild(submitIcon); activation.bindAction('submit', async event => { event.stopPropagation(); @@ -1900,7 +2258,8 @@ class MergeSegmentsTool extends LayerTool { points.classList.add('graphene-merge-segments-merges'); body.appendChild(points); - const segmentWidgetFactory = SegmentWidgetFactory.make(this.layer.displayState, /*includeUnmapped=*/ true); + const segmentWidgetFactory = + SegmentWidgetFactory.make(this.layer.displayState, /*includeUnmapped=*/ true); const makeWidget = (id: Uint64MapEntry) => { const row = segmentWidgetFactory.getWithNormalizedId(id); row.classList.add('neuroglancer-segment-list-entry-double-line'); @@ -1908,7 +2267,7 @@ class MergeSegmentsTool extends LayerTool { }; const createPointElement = (id: Uint64) => { - const containerEl = document.createElement('div'); + const containerEl = document.createElement('div'); containerEl.classList.add('graphene-merge-segments-point') const widget = makeWidget(augmentSegmentId(this.layer.displayState, id)); containerEl.appendChild(widget); @@ -1916,11 +2275,11 @@ class MergeSegmentsTool extends LayerTool { }; const createSubmissionElement = (submission: MergeSubmission) => { - const containerEl = document.createElement('div'); + const containerEl = document.createElement('div'); containerEl.classList.add('graphene-merge-segments-submission'); containerEl.appendChild(createPointElement(submission.sink.rootId)); if (submission.source) { - containerEl.appendChild(document.createElement('div')).textContent = "ꕹ"; + containerEl.appendChild(document.createElement('div')).textContent = 'ꕹ'; containerEl.appendChild(createPointElement(submission.source.rootId)); } if (!submission.locked) { @@ -1934,7 +2293,7 @@ class MergeSegmentsTool extends LayerTool { })); } if (submission.status) { - const statusEl = document.createElement('div'); + const statusEl = document.createElement('div'); statusEl.classList.add('graphene-merge-segments-submission-status'); statusEl.textContent = submission.status; containerEl.appendChild(statusEl); @@ -1963,6 +2322,113 @@ class MergeSegmentsTool extends LayerTool { } } +const FIND_PATH_INPUT_EVENT_MAP = EventActionMap.fromObject({ + 'at:shift?+enter': {action: 'submit'}, + 'at:shift?+control+mousedown0': {action: 'add-point'}, +}); + +class FindPathTool extends LayerTool { + activate(activation: ToolActivation) { + const {layer} = this; + const {graphConnection: {value: graphConnection}} = layer; + if (!graphConnection || !(graphConnection instanceof GraphConnection)) return; + const {state: {findPathState}, findPathAnnotationState} = graphConnection; + const {source, target, precisionMode} = findPathState; + // Ensure we use the same segmentationGroupState while activated. + const segmentationGroupState = this.layer.displayState.segmentationGroupState.value; + const {body, header} = makeToolActivationStatusMessageWithHeader(activation); + header.textContent = 'Find Path'; + body.classList.add('graphene-tool-status', 'graphene-find-path'); + const submitAction = () => { + findPathState.triggerPathUpdate.dispatch(); + }; + body.appendChild(makeIcon({ + text: 'Submit', + title: 'Submit Find Path', + onClick: () => { + submitAction(); + } + })); + body.appendChild(makeIcon({ + text: 'Clear', + title: 'Clear Find Path', + onClick: () => { + findPathState.source.reset(); + findPathState.target.reset(); + findPathState.centroids.reset(); + } + })); + const checkbox = activation.registerDisposer(new TrackableBooleanCheckbox(precisionMode)); + const label = document.createElement('label'); + const labelText = document.createElement('span'); + labelText.textContent = 'Precision mode: '; + label.appendChild(labelText); + label.title = 'Precision mode returns a more accurate path, but takes longer.'; + label.appendChild(checkbox.element); + body.appendChild(label); + const annotationElements = document.createElement('div'); + annotationElements.classList.add('find-path-annotations'); + body.appendChild(annotationElements); + const bindings = getDefaultAnnotationListBindings(); + this.registerDisposer(new MouseEventBinder(annotationElements, bindings)); + const updateAnnotationElements = () => { + removeChildren(annotationElements); + const maxColumnWidths = [0, 0, 0]; + const globalDimensionIndices = [0, 1, 2]; + const localDimensionIndices: number[] = []; + const template = + '[symbol] 2ch [dim] var(--neuroglancer-column-0-width) [dim] var(--neuroglancer-column-1-width) [dim] var(--neuroglancer-column-2-width) [delete] min-content'; + const endpoints = [source, target]; + const endpointAnnotations = + endpoints.map(x => x.value?.annotationReference?.value).filter(x => x) as Annotation[]; + for (const annotation of endpointAnnotations) { + const [element, elementColumnWidths] = makeAnnotationListElement( + this.layer, annotation, findPathAnnotationState, template, globalDimensionIndices, + localDimensionIndices); + for (const [column, width] of elementColumnWidths.entries()) { + maxColumnWidths[column] = width; + } + annotationElements.appendChild(element); + } + for (const [column, width] of maxColumnWidths.entries()) { + annotationElements.style.setProperty( + `--neuroglancer-column-${column}-width`, `${width + 2}ch`); + } + }; + findPathState.changed.add(updateAnnotationElements); + updateAnnotationElements(); + activation.bindInputEventMap(FIND_PATH_INPUT_EVENT_MAP); + activation.bindAction('submit', event => { + event.stopPropagation(); + submitAction(); + }); + activation.bindAction('add-point', event => { + event.stopPropagation(); + (async () => { + if (!source.value) { // first selection + const selection = maybeGetSelection(this, segmentationGroupState.visibleSegments); + if (selection) { + source.value = selection; + } + } else if (!target.value) { + const selection = maybeGetSelection(this, segmentationGroupState.visibleSegments); + if (selection) { + target.value = selection; + } + } + })(); + }); + } + + toJSON() { + return GRAPHENE_FIND_PATH_TOOL_ID; + } + + get description() { + return `find path`; + } +} + registerTool(SegmentationUserLayer, GRAPHENE_MULTICUT_SEGMENTS_TOOL_ID, layer => { return new MulticutSegmentsTool(layer, true); }); @@ -1971,6 +2437,10 @@ registerTool(SegmentationUserLayer, GRAPHENE_MERGE_SEGMENTS_TOOL_ID, layer => { return new MergeSegmentsTool(layer, true); }); +registerTool(SegmentationUserLayer, GRAPHENE_FIND_PATH_TOOL_ID, layer => { + return new FindPathTool(layer, true); +}); + const ANNOTATE_MERGE_LINE_TOOL_ID = 'annotateMergeLine'; registerLegacyTool( diff --git a/src/neuroglancer/datasource/graphene/graphene.css b/src/neuroglancer/datasource/graphene/graphene.css index 4c682eb969..6fdea0967e 100644 --- a/src/neuroglancer/datasource/graphene/graphene.css +++ b/src/neuroglancer/datasource/graphene/graphene.css @@ -10,12 +10,12 @@ color: #4444ff; } -.graphene-multicut-status { +.graphene-tool-status { display: flex; - flex-direction: row; + gap: 10px; } -.graphene-multicut-status > .activeGroupIndicator { +.graphene-multicut > .activeGroupIndicator { padding: 2px; margin: auto 0; background-color: red; @@ -23,15 +23,15 @@ font-weight: 900; } -.graphene-multicut-status > .activeGroupIndicator.blueGroup { +.graphene-multicut > .activeGroupIndicator.blueGroup { background-color: blue; } -.graphene-multicut-status > .activeGroupIndicator::after { +.graphene-multicut > .activeGroupIndicator::after { content: "Red"; } -.graphene-multicut-status > .activeGroupIndicator.blueGroup::after { +.graphene-multicut > .activeGroupIndicator.blueGroup::after { content: "Blue"; } @@ -45,12 +45,8 @@ gap: 10px; } -.graphene-merge-segments-status { - display: flex; - gap: 10px; -} - -.graphene-merge-segments-status .neuroglancer-icon, .graphene-multicut-status .neuroglancer-icon { +.graphene-tool-status .neuroglancer-icon + { height: 100%; } @@ -72,3 +68,22 @@ .graphene-merge-segments-point .neuroglancer-segment-list-entry-copy-container { display: none; } + +.graphene-find-path > label { + display: flex; +} + +.graphene-find-path > label > span { + display: flex; + align-content: center; + flex-wrap: wrap; +} + +.find-path-annotations { + display: flex; + gap: 10px; +} + +.find-path-annotations > .neuroglancer-annotation-list-entry { + background-color: black; +} diff --git a/src/neuroglancer/status.ts b/src/neuroglancer/status.ts index a371119bfa..20f633d2f5 100644 --- a/src/neuroglancer/status.ts +++ b/src/neuroglancer/status.ts @@ -29,6 +29,7 @@ export class StatusMessage { if (statusContainer === null) { statusContainer = document.createElement('ul'); statusContainer.id = 'statusContainer'; + statusContainer.tabIndex = -1; const el: HTMLElement | null = document.getElementById('neuroglancer-container'); if (el) { el.appendChild(statusContainer); diff --git a/src/neuroglancer/ui/annotations.ts b/src/neuroglancer/ui/annotations.ts index 16a9476817..82c93d9d2c 100644 --- a/src/neuroglancer/ui/annotations.ts +++ b/src/neuroglancer/ui/annotations.ts @@ -61,6 +61,8 @@ import {makeMoveToButton} from 'neuroglancer/widget/move_to_button'; import {Tab} from 'neuroglancer/widget/tab_view'; import {VirtualList, VirtualListSource} from 'neuroglancer/widget/virtual_list'; + + export class MergedAnnotationStates extends RefCounted implements WatchableValueInterface { changed = new NullarySignal(); @@ -461,7 +463,24 @@ export class AnnotationLayerView extends Tab { private render(index: number) { const {annotation, state} = this.listElements[index]; - return this.makeAnnotationListElement(annotation, state); + const {layer, gridTemplate, globalDimensionIndices, localDimensionIndices} = this; + const [element, elementColumnWidths] = makeAnnotationListElement(layer, annotation, state, gridTemplate, globalDimensionIndices, localDimensionIndices); + for (const [column, width] of elementColumnWidths.entries()) { + this.setColumnWidth(column, width); + } + element.addEventListener('mouseenter', () => { + this.displayState.hoverState.value = { + id: annotation.id, + partIndex: 0, + annotationLayerState: state, + }; + }); + const selectionState = this.selectedAnnotationState.value; + if (selectionState !== undefined && selectionState.annotationLayerState === state && + selectionState.annotationId === annotation.id) { + element.classList.add('neuroglancer-annotation-selected'); + } + return element; } private setColumnWidth(column: number, width: number) { @@ -649,120 +668,6 @@ export class AnnotationLayerView extends Tab { this.updateHoverView(); this.updateSelectionView(); } - - private makeAnnotationListElement(annotation: Annotation, state: AnnotationLayerState) { - const chunkTransform = state.chunkTransform.value as ChunkTransformParameters; - const element = document.createElement('div'); - element.classList.add('neuroglancer-annotation-list-entry'); - element.dataset.color = state.displayState.color.toString(); - element.style.gridTemplateColumns = this.gridTemplate; - const icon = document.createElement('div'); - icon.className = 'neuroglancer-annotation-icon'; - icon.textContent = annotationTypeHandlers[annotation.type].icon; - element.appendChild(icon); - - let deleteButton: HTMLElement|undefined; - - const maybeAddDeleteButton = () => { - if (state.source.readonly) return; - if (deleteButton !== undefined) return; - deleteButton = makeDeleteButton({ - title: 'Delete annotation', - onClick: event => { - event.stopPropagation(); - event.preventDefault(); - const ref = state.source.getReference(annotation.id); - try { - state.source.delete(ref); - } finally { - ref.dispose(); - } - }, - }); - deleteButton.classList.add('neuroglancer-annotation-list-entry-delete'); - element.appendChild(deleteButton); - }; - - let numRows = 0; - visitTransformedAnnotationGeometry(annotation, chunkTransform, (layerPosition, isVector) => { - isVector; - ++numRows; - const position = document.createElement('div'); - position.className = 'neuroglancer-annotation-position'; - element.appendChild(position); - let i = 0; - const addDims = - (viewDimensionIndices: readonly number[], layerDimensionIndices: readonly number[]) => { - for (const viewDim of viewDimensionIndices) { - const layerDim = layerDimensionIndices[viewDim]; - if (layerDim !== -1) { - const coord = Math.floor(layerPosition[layerDim]); - const coordElement = document.createElement('div'); - const text = coord.toString() - coordElement.textContent = text; - coordElement.classList.add('neuroglancer-annotation-coordinate'); - coordElement.style.gridColumn = `dim ${i + 1}`; - this.setColumnWidth(i, text.length); - position.appendChild(coordElement); - } - ++i; - } - }; - addDims( - this.globalDimensionIndices, chunkTransform.modelTransform.globalToRenderLayerDimensions); - addDims( - this.localDimensionIndices, chunkTransform.modelTransform.localToRenderLayerDimensions); - maybeAddDeleteButton(); - }); - if (annotation.description) { - ++numRows; - const description = document.createElement('div'); - description.classList.add('neuroglancer-annotation-description'); - description.textContent = annotation.description; - element.appendChild(description); - } - icon.style.gridRow = `span ${numRows}`; - if (deleteButton !== undefined) { - deleteButton.style.gridRow = `span ${numRows}`; - } - element.addEventListener('mouseenter', () => { - this.displayState.hoverState.value = { - id: annotation.id, - partIndex: 0, - annotationLayerState: state, - }; - this.layer.selectAnnotation(state, annotation.id, false); - }); - element.addEventListener('action:select-position', event => { - event.stopPropagation(); - this.layer.selectAnnotation(state, annotation.id, 'toggle'); - }); - - element.addEventListener('action:pin-annotation', event => { - event.stopPropagation(); - this.layer.selectAnnotation(state, annotation.id, true); - }); - - element.addEventListener('action:move-to-annotation', event => { - event.stopPropagation(); - event.preventDefault(); - const {layerRank} = chunkTransform; - const chunkPosition = new Float32Array(layerRank); - const layerPosition = new Float32Array(layerRank); - getCenterPosition(chunkPosition, annotation); - matrix.transformPoint( - layerPosition, chunkTransform.chunkToLayerTransform, layerRank + 1, chunkPosition, - layerRank); - setLayerPosition(this.layer, chunkTransform, layerPosition); - }); - - const selectionState = this.selectedAnnotationState.value; - if (selectionState !== undefined && selectionState.annotationLayerState === state && - selectionState.annotationId === annotation.id) { - element.classList.add('neuroglancer-annotation-selected'); - } - return element; - } } export class AnnotationTab extends Tab { @@ -1670,3 +1575,107 @@ export function UserLayerWithAnnotationsMixin>; + +export function makeAnnotationListElement(layer: UserLayerWithAnnotations, annotation: Annotation, state: AnnotationLayerState, gridTemplate: string, globalDimensionIndices: number[], localDimensionIndices: number[]): [HTMLDivElement, number[]] { + const chunkTransform = state.chunkTransform.value as ChunkTransformParameters; + const element = document.createElement('div'); + element.classList.add('neuroglancer-annotation-list-entry'); + element.dataset.color = state.displayState.color.toString(); + element.style.gridTemplateColumns = gridTemplate; + const icon = document.createElement('div'); + icon.className = 'neuroglancer-annotation-icon'; + icon.textContent = annotationTypeHandlers[annotation.type].icon; + element.appendChild(icon); + + let deleteButton: HTMLElement|undefined; + + const maybeAddDeleteButton = () => { + if (state.source.readonly) return; + if (deleteButton !== undefined) return; + deleteButton = makeDeleteButton({ + title: 'Delete annotation', + onClick: event => { + event.stopPropagation(); + event.preventDefault(); + const ref = state.source.getReference(annotation.id); + try { + state.source.delete(ref); + } finally { + ref.dispose(); + } + }, + }); + deleteButton.classList.add('neuroglancer-annotation-list-entry-delete'); + element.appendChild(deleteButton); + }; + + const columnWidths: number[] = []; + + let numRows = 0; + visitTransformedAnnotationGeometry(annotation, chunkTransform, (layerPosition, isVector) => { + isVector; + ++numRows; + const position = document.createElement('div'); + position.className = 'neuroglancer-annotation-position'; + element.appendChild(position); + let i = 0; + + const addDims = + (viewDimensionIndices: readonly number[], layerDimensionIndices: readonly number[]) => { + for (const viewDim of viewDimensionIndices) { + const layerDim = layerDimensionIndices[viewDim]; + if (layerDim !== -1) { + const coord = Math.floor(layerPosition[layerDim]); + const coordElement = document.createElement('div'); + const text = coord.toString(); + coordElement.textContent = text; + coordElement.classList.add('neuroglancer-annotation-coordinate'); + coordElement.style.gridColumn = `dim ${i + 1}`; + columnWidths[i] = Math.max(columnWidths[i] || 0, text.length); + position.appendChild(coordElement); + } + ++i; + } + }; + addDims( + globalDimensionIndices, chunkTransform.modelTransform.globalToRenderLayerDimensions); + addDims( + localDimensionIndices, chunkTransform.modelTransform.localToRenderLayerDimensions); + maybeAddDeleteButton(); + }); + if (annotation.description) { + ++numRows; + const description = document.createElement('div'); + description.classList.add('neuroglancer-annotation-description'); + description.textContent = annotation.description; + element.appendChild(description); + } + icon.style.gridRow = `span ${numRows}`; + if (deleteButton !== undefined) { + deleteButton.style.gridRow = `span ${numRows}`; + } + element.addEventListener('mouseenter', () => { + layer.selectAnnotation(state, annotation.id, false); + }); + element.addEventListener('action:select-position', event => { + event.stopPropagation(); + layer.selectAnnotation(state, annotation.id, 'toggle'); + }); + element.addEventListener('action:pin-annotation', event => { + event.stopPropagation(); + layer.selectAnnotation(state, annotation.id, true); + }); + element.addEventListener('action:move-to-annotation', event => { + event.stopPropagation(); + event.preventDefault(); + const {layerRank} = chunkTransform; + const chunkPosition = new Float32Array(layerRank); + const layerPosition = new Float32Array(layerRank); + getCenterPosition(chunkPosition, annotation); + matrix.transformPoint( + layerPosition, chunkTransform.chunkToLayerTransform, layerRank + 1, chunkPosition, + layerRank); + setLayerPosition(layer, chunkTransform, layerPosition); + }); + return [element, columnWidths]; + } \ No newline at end of file diff --git a/src/neuroglancer/util/json.ts b/src/neuroglancer/util/json.ts index 8bb862ee1b..6425c7b4b6 100644 --- a/src/neuroglancer/util/json.ts +++ b/src/neuroglancer/util/json.ts @@ -616,6 +616,16 @@ export function verifyIntegerArray(a: unknown) { return a; } +export function verifyFloatArray(a: unknown) { + if (!Array.isArray(a)) { + throw new Error(`Expected array, received: ${JSON.stringify(a)}.`); + } + for (let x of a) { + verifyFloat(x); + } + return a; +} + export function verifyBoolean(x: any) { if (typeof x !== 'boolean') { throw new Error(`Expected boolean, received: ${JSON.stringify(x)}`);