threeSurfer.component.ts

import { Component, Output, EventEmitter, ElementRef, OnDestroy, AfterViewInit, Inject, Optional, ChangeDetectionStrategy } from "@angular/core";
import { EnumViewerEvt, IViewer, TViewerEvent } from "src/viewerModule/viewer.interface";
import { combineLatest, forkJoin, from, merge, Observable, Subject } from "rxjs";
import { debounceTime, distinctUntilChanged, filter, map, scan, shareReplay, switchMap } from "rxjs/operators";
import { ComponentStore } from "src/viewerModule/componentStore";
import { select, Store } from "@ngrx/store";
import { ClickInterceptor, CLICK_INTERCEPTOR_INJECTOR } from "src/util";
import { MatSnackBar } from "@angular/material/snack-bar";
import { CONST } from 'common/constants'
import { getUuid } from "src/util/fn";
import { AUTO_ROTATE, TInteralStatePayload, ViewerInternalStateSvc } from "src/viewerModule/viewerInternalState.service";
import { atlasAppearance, atlasSelection } from "src/state";
import { ThreeSurferCustomLabelLayer, ThreeSurferCustomLayer, ColorMapCustomLayer } from "src/state/atlasAppearance/const";
import { SxplrRegion } from "src/atlasComponents/sapi/type_sxplr"
import { arrayEqual } from "src/util/array";
import { ThreeSurferEffects } from "../store/effects";
import { selectors, actions  } from "../store"
import { SAPI } from "src/atlasComponents/sapi";

const viewerType = 'ThreeSurfer'
type TInternalState = {
  camera: {
    x: number
    y: number
    z: number
  }
  mode: string
  hemisphere: 'left' | 'right' | 'both'
}
const pZoomFactor = 5e3

type THandlingCustomEv = {
  regions: SxplrRegion[]
  error?: string
  evMesh?: {
    faceIndex: number
    verticesIndicies: number[]
  }
}

type TCameraOrientation = {
  perspectiveOrientation: number[]
  perspectiveZoom: number
}

type TThreeGeometry = {
  visible: boolean
}
type GiiInstance = unknown
type TThreeSurfer = {
  loadMesh: (url: string) => Promise<TThreeGeometry>
  unloadMesh: (geom: TThreeGeometry) => void
  redraw: (geom: TThreeGeometry) => void
  applyColorMap: (geom: TThreeGeometry, idxMap?: number[], custom?: { usePreset?: any, custom?: Map<number, number[]> }) => void
  loadColormap: (url: string) => Promise<GiiInstance>
  setupAnimation: () => void
  dispose: () => void
  control: any
  camera: any
  customColormap: WeakMap<TThreeGeometry, any>
}

type LateralityRecord<T> = Record<string, T>

const threshold = 1e-3

function cameraNavsAreSimilar(c1: TCameraOrientation, c2: TCameraOrientation){
  if (c1 === c2) return true
  if (!!c1 && !!c2) return true

  if (!c1 && !!c2) return false
  if (!c2 && !!c1) return false

  if (Math.abs(c1.perspectiveZoom - c2.perspectiveZoom) > threshold) return false
  if ([0, 1, 2, 3].some(
    idx => Math.abs(c1.perspectiveOrientation[idx] - c2.perspectiveOrientation[idx]) > threshold
  )) {
    return false
  }
  return true
}

@Component({
  selector: 'three-surfer-glue-cmp',
  templateUrl: './threeSurfer.template.html',
  styleUrls: [
    './threeSurfer.style.css'
  ],
  providers: [ ComponentStore ],
  changeDetection: ChangeDetectionStrategy.OnPush
})

export class ThreeSurferGlueCmp implements IViewer<'threeSurfer'>, AfterViewInit, OnDestroy {

  
  @Output()
  viewerEvent = new EventEmitter<TViewerEvent<'threeSurfer'>>()

  private domEl: HTMLElement
  private mainStoreCameraNav: TCameraOrientation = null
  private localCameraNav: TCameraOrientation = null

  public lateralityMeshRecord: LateralityRecord<{
    visible: boolean
    meshLayer: ThreeSurferCustomLayer
    mesh: TThreeGeometry
  }> = {}

  public latLblIdxRecord: LateralityRecord<{
    indexLayer: ThreeSurferCustomLabelLayer
    labelIndices: number[]
  }> = {}
  private internalStateNext: (arg: TInteralStatePayload<TInternalState>) => void

  private mouseoverRegions: SxplrRegion[] = []
  
  private selectedRegions$ = this.store$.pipe(
    select(atlasSelection.selectors.selectedRegions)
  )

  private customLayers$ = this.store$.pipe(
    select(atlasAppearance.selectors.customLayers),
    distinctUntilChanged(arrayEqual((o, n) => o.id === n.id)),
    shareReplay(1)
  )
  public meshLayers$: Observable<ThreeSurferCustomLayer[]> = this.customLayers$.pipe(
    map(layers => layers.filter(l => l.clType === "baselayer/threesurfer") as ThreeSurferCustomLayer[]),
    distinctUntilChanged(arrayEqual((o, n) => o.id === n.id)),
  )

  private vertexIndexLayers$: Observable<ThreeSurferCustomLabelLayer[]> = this.customLayers$.pipe(
    map(layers => layers.filter(l => l.clType === "baselayer/threesurfer-label") as ThreeSurferCustomLabelLayer[]),
    distinctUntilChanged(arrayEqual((o, n) => o.id === n.id)),
  )

  /**
   * maps laterality to label index to sapi region
   */
  private latLblIdxToRegionRecord: LateralityRecord<Record<number, SxplrRegion>> = {}
  private latLblIdxToRegionRecord$: Observable<LateralityRecord<Record<number, SxplrRegion>>> = combineLatest([
    this.store$.pipe(
      atlasSelection.fromRootStore.distinctATP()
    ),
    this.store$.pipe(
      select(atlasSelection.selectors.selectedParcAllRegions),
    )
  ]).pipe(
    switchMap(([ { atlas, parcellation, template },  regions]) => {
      const returnObj = {
        'left': {} as Record<number, SxplrRegion>,
        'right': {} as Record<number, SxplrRegion>
      }
      return merge(
        ...regions.map(region => 
          from(this.sapi.getRegionLabelIndices(template, parcellation, region)).pipe(
            map(label => ({ region, label }))
          )
        )
      ).pipe(
        scan((acc, curr) => {
          const { label, region } = curr
          
          let key : 'left' | 'right'
          if ( /left/i.test(region.name) ) key = 'left'
          if ( /right/i.test(region.name) ) key = 'right'
          if (!key) {
            /**
             * TODO
             * there are ... more regions than expected, which has label index without laterality
             */
            return { ...acc }
          }
          return {
            ...acc,
            [key]: {
              ...acc[key],
              [label]: region
            }
          }
        }, {'left': {}, 'right': {}})
      )
    })
  )

  /**
   * colormap in use (both base & custom)
   */

  private colormapInUse: ColorMapCustomLayer
  private colormaps$: Observable<ColorMapCustomLayer[]> = this.customLayers$.pipe(
    map(layers => layers.filter(l => l.clType === "baselayer/colormap" || l.clType === "customlayer/colormap") as ColorMapCustomLayer[]),
  )

  /**
   * show delination map
   */
  private showDelineation: boolean = true

  public threeSurferSurfaceVariants$ = this.effect.onATPDebounceThreeSurferLayers$.pipe(
    map(({ surfaces }) => surfaces.reduce((acc, val) => acc.includes(val.variant) ? acc : [...acc, val.variant] ,[] as string[]))
  )
  public selectedSurfaceLayerId$ = this.store$.pipe(
    select(selectors.getSelectedSurfaceVariant)
  )

  constructor(
    private effect: ThreeSurferEffects,
    private el: ElementRef,
    private store$: Store,
    private navStateStoreRelay: ComponentStore<{ perspectiveOrientation: [number, number, number, number], perspectiveZoom: number }>,
    private sapi: SAPI,
    private snackbar: MatSnackBar,
    @Optional() intViewerStateSvc: ViewerInternalStateSvc,
    @Optional() @Inject(CLICK_INTERCEPTOR_INJECTOR) clickInterceptor: ClickInterceptor,
  ){
    if (intViewerStateSvc) {
      const {
        done,
        next,
      } = intViewerStateSvc.registerEmitter({
        "@type": 'TViewerInternalStateEmitter',
        viewerType,
        applyState: arg => {
          if (arg.viewerType === AUTO_ROTATE) {
            const autoPlayFlag = (arg.payload as any).play
            const reverseFlag = (arg.payload as any).reverse
            const autoplaySpeed = (arg.payload as any).speed
            this.toTsRef(tsRef => {
              tsRef.control.autoRotate = autoPlayFlag
              tsRef.control.autoRotateSpeed = autoplaySpeed * (reverseFlag ? -1 : 1)
            })
            return
          }
          if (arg.viewerType !== viewerType) return
          this.toTsRef(tsRef => {
            tsRef.camera.position.copy((arg.payload as any).camera)
          })
        }
      })
      this.internalStateNext = next
      this.onDestroyCb.push(() => done())
    }

    /**
     * intercept click and act
     */
    if (clickInterceptor) {
      const handleClick = (ev: MouseEvent) => {

        // if does not click inside container, ignore
        if (!(this.el.nativeElement as HTMLElement).contains(ev.target as HTMLElement)) {
          return true
        }
        
        if (this.mouseoverRegions.length === 0) return true
        if (this.mouseoverRegions.length > 1) {
          this.snackbar.open(CONST.DOES_NOT_SUPPORT_MULTI_REGION_SELECTION, 'Dismiss', {
            duration: 3000
          })
          return true
        }

        const regions = this.mouseoverRegions.slice(0, 1) as any[]
        this.store$.dispatch(
          atlasSelection.actions.setSelectedRegions({ regions })
        )
        return true
      }
      const { register, deregister } = clickInterceptor
      register(handleClick)
      this.onDestroyCb.push(
        () => { deregister(register) }
      )
    }
    
    this.domEl = this.el.nativeElement

    /**
     * subscribe to camera custom event
     */
    const cameraSub = this.cameraEv$.pipe(
      filter(v => !!v),
      debounceTime(160)
    ).subscribe(() => {
      
      const THREE = (window as any).ThreeSurfer.THREE
      
      const q = new THREE.Quaternion()
      const t = new THREE.Vector3()
      const s = new THREE.Vector3()

      /**
       * ThreeJS interpretes the scene differently to neuroglancer in subtle ways. 
       * At [0, 0, 0, 1] decomposed camera quaternion, for example,
       * - ThreeJS: view from superior -> inferior, anterior as top, right hemisphere as right
       * - NG: view from from inferior -> superior, posterior as top, left hemisphere as right
       * 
       * multiplying the exchange factor [-1, 0, 0, 0] converts ThreeJS convention to NG convention
       */
      const cameraM = this.tsRef.camera.matrix
      cameraM.decompose(t, q, s)
      const exchangeFactor = new THREE.Quaternion(-1, 0, 0, 0)

      try {
        this.navStateStoreRelay.setState({
          perspectiveOrientation: q.multiply(exchangeFactor).toArray(),
          perspectiveZoom: t.length()
        })
      } catch (_e) {
        // LockError, ignore
      }
    })

    this.onDestroyCb.push(
      () => cameraSub.unsubscribe()
    )

    /**
     * subscribe to navstore relay store and negotiate setting global state
     */
    const navStateSub = this.navStateStoreRelay.select(s => s).subscribe(v => {
      this.store$.dispatch(
        atlasSelection.actions.setNavigation({
          navigation: {
            position: [0, 0, 0],
            orientation: [0, 0, 0, 1],
            zoom: 1e6,
            perspectiveOrientation: v.perspectiveOrientation,
            perspectiveZoom: v.perspectiveZoom * pZoomFactor
          }
        })
      )
    })

    this.onDestroyCb.push(
      () => navStateSub.unsubscribe()
    )

    /**
     * subscribe to main store and negotiate with relay to set camera
     */
    const navSub = this.store$.pipe(
      select(atlasSelection.selectors.navigation),
      filter(v => !!v),
    ).subscribe(nav => {
      const { perspectiveOrientation, perspectiveZoom } = nav
      this.mainStoreCameraNav = {
        perspectiveOrientation,
        perspectiveZoom
      }

      if (!cameraNavsAreSimilar(this.mainStoreCameraNav, this.localCameraNav)) {
        this.relayStoreLock = this.navStateStoreRelay.getLock()
        const THREE = (window as any).ThreeSurfer.THREE
        
        const cameraQuat = new THREE.Quaternion(...this.mainStoreCameraNav.perspectiveOrientation)
        const cameraPos = new THREE.Vector3(0, 0, this.mainStoreCameraNav.perspectiveZoom / pZoomFactor)
        
        /**
         * ThreeJS interpretes the scene differently to neuroglancer in subtle ways. 
         * At [0, 0, 0, 1] decomposed camera quaternion, for example,
         * - ThreeJS: view from superior -> inferior, anterior as top, right hemisphere as right
         * - NG: view from from inferior -> superior, posterior as top, left hemisphere as right
         * 
         * multiplying the exchange factor [-1, 0, 0, 0] converts ThreeJS convention to NG convention
         */
        const exchangeFactor = new THREE.Quaternion(-1, 0, 0, 0)
        cameraQuat.multiply(exchangeFactor)

        cameraPos.applyQuaternion(cameraQuat)
        this.toTsRef(tsRef => {
          tsRef.camera.position.copy(cameraPos)
          if (this.relayStoreLock) this.relayStoreLock()
        })
      }
    })

    this.onDestroyCb.push(
      () => navSub.unsubscribe()
    )
  }

  private tsRef: TThreeSurfer
  private selectedRegions: SxplrRegion[] = []

  private relayStoreLock: () => void = null
  private tsRefInitCb: ((tsRef: any) => void)[] = []
  private toTsRef(callback: (tsRef: any) => void) {
    if (this.tsRef) {
      callback(this.tsRef)
      return
    }
    this.tsRefInitCb.push(callback)
  }

  private async loadMeshes(layers: ThreeSurferCustomLayer[]) {
    if (!this.tsRef) throw new Error(`loadMeshes error: this.tsRef is not defined!!`)

    /**
     * remove the layers... 
     */
    for (const layer of layers) {
      if (!!this.lateralityMeshRecord[layer.laterality]) {
        this.tsRef.unloadMesh(this.lateralityMeshRecord[layer.laterality].mesh)
      }
    }

    for (const layer of layers) {
      const threeMesh = await this.tsRef.loadMesh(layer.source)
      this.lateralityMeshRecord[layer.laterality] = {
        visible: true,
        meshLayer: layer,
        mesh: threeMesh
      }
    }
    this.applyColor()
  }

  private async loadVertexIndexMap(layers: ThreeSurferCustomLabelLayer[]) {
    if (!this.tsRef) throw new Error(`loadVertexIndexMap error: this.tsRef is not defined!!`)
    for (const layer of layers) {
      const giiInstance = await this.tsRef.loadColormap(layer.source)

      let labelIndices: number[] = giiInstance[0].getData()
      if (giiInstance[0].attributes.DataType === 'NIFTI_TYPE_INT16') {
        labelIndices = (window as any).ThreeSurfer.GiftiBase.castF32UInt16(labelIndices)
      }
      this.latLblIdxRecord[layer.laterality] = {
        indexLayer: layer,
        labelIndices
      }
    }
    this.applyColor()
  }

  private applyColor() {
    /**
     * on apply color map, reset mesh visibility
     * this issue is more difficult to solve than first anticiplated.
     * test scenarios:
     * 
     * 1/ hide hemisphere, select region
     * 2/ hide hemisphere, select region, unhide hemisphere
     * 3/ select region, hide hemisphere, deselect region
     */
    if (!this.colormapInUse) return
    if (!this.tsRef) return
    
    const isBaseCM = this.colormapInUse?.clType === "baselayer/colormap"

    for (const laterality in this.lateralityMeshRecord) {
      const { mesh } = this.lateralityMeshRecord[laterality]
      if (!this.latLblIdxRecord[laterality]) continue
      const { labelIndices } = this.latLblIdxRecord[laterality]

      const lblIdxToRegionRecord = this.latLblIdxToRegionRecord[laterality]
      if (!lblIdxToRegionRecord) {
        this.tsRef.applyColorMap(mesh, labelIndices)
        continue
      }
      const map = new Map<number, number[]>()
      for (const lblIdx in lblIdxToRegionRecord) {
        const region = lblIdxToRegionRecord[lblIdx]
        let color: number[]
        if (!this.showDelineation) {
          color = [1,1,1]
        } else if (isBaseCM && this.selectedRegions.length > 0 && !this.selectedRegions.includes(region)) {
          color = [1,1,1]
        } else {
          color = (this.colormapInUse.colormap.get(region) || [255, 255, 255]).map(v => v/255)
        }
        map.set(Number(lblIdx), color)
      }
      this.tsRef.applyColorMap(mesh, labelIndices, {
        custom: map
      })
    }
  }

  private handleCustomMouseEv(detail: any){
    const evMesh = detail.mesh && {
      faceIndex: detail.mesh.faceIndex,
      // typo in three-surfer
      verticesIndicies: detail.mesh.verticesIdicies
    }
    const custEv: THandlingCustomEv = {
      regions: [],
      evMesh
    }
    
    if (!detail.mesh) {
      return this.handleMouseoverEvent(custEv)
    }

    const {
      geometry: evGeometry,
      // typo in three-surfer
      verticesIdicies: evVerticesIndicies,
    } = detail.mesh as { geometry: TThreeGeometry, verticesIdicies: number[] }

    for (const laterality in this.lateralityMeshRecord) {
      const meshRecord = this.lateralityMeshRecord[laterality]
      if (meshRecord.mesh !== evGeometry) {
        continue
      }
      /**
       * if either labelindex record or colormap record is undefined for this laterality, emit empty event
       */
      if (!this.latLblIdxRecord[laterality] || !this.latLblIdxToRegionRecord[laterality]) {
        return this.handleMouseoverEvent(custEv)
      }
      const labelIndexRecord = this.latLblIdxRecord[laterality]
      const regionRecord = this.latLblIdxToRegionRecord[laterality]

      /**
       * check if the mesh is toggled off
       * if so, do not proceed
       */
      if (!meshRecord.visible) {
        return
      }

      /**
       * translate vertex indices to label indicies via set, to remove duplicates
       */
      const labelIndexSet = new Set<number>()
      for (const idx of evVerticesIndicies){
        const labelOfInterest = labelIndexRecord.labelIndices[idx]
        if (!labelOfInterest) {
          continue
        }
        labelIndexSet.add(labelOfInterest)
      }

      /**
       * decode label index to region
       */
      if (labelIndexSet.size === 0) {
        return this.handleMouseoverEvent(custEv)
      }
      for (const labelIndex of Array.from(labelIndexSet)) {
        if (!regionRecord[labelIndex]) {
          custEv.error = `${custEv.error || ''} Cannot decode label index ${labelIndex}`
          continue
        }
        const region = regionRecord[labelIndex]
        custEv.regions.push(region)
      }

      /**
       * return handle event
       */
      return this.handleMouseoverEvent(custEv)
    }
  }

  private cameraEv$ = new Subject<{ position: { x: number, y: number, z: number }, zoom: number }>()
  private handleCustomCameraEvent(detail: any){
    if (this.internalStateNext) {
      this.internalStateNext({
        "@id": getUuid(),
        "@type": 'TViewerInternalStateEmitterEvent',
        viewerType,
        payload: {
          mode: '',
          camera: detail.position,
          hemisphere: 'both'
        }
      })
    }
    this.cameraEv$.next(detail)
  }

  ngAfterViewInit(): void{
    const customEvHandler = (ev: CustomEvent) => {
      const { type, data } = ev.detail
      if (type === 'mouseover') {
        return this.handleCustomMouseEv(data)
      }
      if (type === 'camera') {
        return this.handleCustomCameraEvent(data)
      }
    }
    this.domEl.addEventListener((window as any).ThreeSurfer.CUSTOM_EVENTNAME_UPDATED, customEvHandler)
    this.onDestroyCb.push(
      () => this.domEl.removeEventListener((window as any).ThreeSurfer.CUSTOM_EVENTNAME_UPDATED, customEvHandler)
    )
    this.tsRef = new (window as any).ThreeSurfer(this.domEl, {highlightHovered: true})

    this.onDestroyCb.push(
      () => {
        this.tsRef.dispose()
        this.tsRef = null
      }
    )
    this.tsRef.control.enablePan = false
    while (this.tsRefInitCb.length > 0) {
      const tsCb = this.tsRefInitCb.pop()
      tsCb(this.tsRef)
    }

    const meshSub = this.meshLayers$.pipe(
      distinctUntilChanged(),
      debounceTime(16),
    ).subscribe(layers => {
      this.loadMeshes(layers)
    })
    const vertexIdxSub = this.vertexIndexLayers$.subscribe(layers => this.loadVertexIndexMap(layers))
    const roiSelectedSub = this.selectedRegions$.subscribe(regions => {
      this.selectedRegions = regions
      this.applyColor()
    })
    const colormapSub = this.colormaps$.subscribe(cm => {
      this.colormapInUse = cm[0] || null
      this.applyColor()
    })
    const recordToRegionSub = this.latLblIdxToRegionRecord$.subscribe(val => this.latLblIdxToRegionRecord = val)
    const hideDelineationSub = this.store$.pipe(
      select(atlasAppearance.selectors.showDelineation)
    ).subscribe(flag => {
      this.showDelineation = flag
      this.applyColor()
      /**
       * apply color resets mesh visibility
       */
      this.updateMeshVisibility()
    })

    this.onDestroyCb.push(() => {
      meshSub.unsubscribe()
      vertexIdxSub.unsubscribe()
      roiSelectedSub.unsubscribe()
      colormapSub.unsubscribe()
      recordToRegionSub.unsubscribe()
      hideDelineationSub.unsubscribe()
    })

    this.viewerEvent.emit({
      type: EnumViewerEvt.VIEWERLOADED,
      data: true
    })
  }

  public mouseoverText: string
  private handleMouseoverEvent(ev: THandlingCustomEv){
    const { regions: mouseover, evMesh, error } = ev
    this.mouseoverRegions = mouseover
    this.viewerEvent.emit({
      type: EnumViewerEvt.VIEWER_CTX,
      data: {
        viewerType: 'threeSurfer',
        payload: {
          fsversion: '',
          faceIndex: evMesh?.faceIndex,
          vertexIndices: evMesh?.verticesIndicies,
          position: [],
          regions: mouseover,
          error
        }
      }
    })
    this.mouseoverText = ''
    if (mouseover.length > 0) {
      this.mouseoverText += mouseover.map(el => el.name).join(' / ')
    }
    if (error) {
      this.mouseoverText += `::error: ${error}`
    }
    if (this.mouseoverText === '') this.mouseoverText = null
  }

  public updateMeshVisibility(): void{

    for (const key in this.lateralityMeshRecord) {

      const latMeshRecord = this.lateralityMeshRecord[key]
      if (!latMeshRecord) {
        return
      }
      const meshObj = this.tsRef.customColormap.get(latMeshRecord.mesh)
      if (!meshObj) {
        throw new Error(`mesh obj not found!`)
      }
      meshObj.mesh.visible = latMeshRecord.visible
    }
  }

  switchSurfaceLayer(variant: string): void{
    this.store$.dispatch(
      actions.selectSurfaceVariant({
        variant
      })
    )
  }

  private onDestroyCb: (() => void) [] = []

  ngOnDestroy(): void {
    while (this.onDestroyCb.length > 0) this.onDestroyCb.pop()()
  }
}