Skip to content

Commit

Permalink
feat(octra): dynamically move transcription units
Browse files Browse the repository at this point in the history
  • Loading branch information
@julianpoemp
julianpoemp committed Oct 24, 2023
1 parent 67a5ea6 commit 0af071e
Show file tree
Hide file tree
Showing 5 changed files with 146 additions and 32 deletions.
2 changes: 1 addition & 1 deletion apps/octra/src/app/core/modals/tools-modal/tools-modal.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ <h5 class="modal-title" id="modal-title">
</button>
<div
*ngIf="tools.audioCutting.status !== 'idle'"
class="mb-2"
class="my-2"
>
<div class="progress" style="height: 20px">
<div
Expand Down
1 change: 1 addition & 0 deletions apps/octra/src/app/editors/2D-editor/2D-editor.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
[refreshOnInternChanges]="false"
[annotation]="annotationStoreService.transcript!"
(currentLevelChange)="onCurrentLevelChange($event)"
(boundaryDragging)="onBoundaryDragged($event)"
id="special"
style="height: 400px; display: flex; flex: auto; flex-direction: column"
>
Expand Down
4 changes: 4 additions & 0 deletions apps/octra/src/app/editors/2D-editor/2D-editor.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1092,4 +1092,8 @@ export class TwoDEditorComponent
);
}
}

onBoundaryDragged($event: any) {
console.log($event);
}
}
24 changes: 15 additions & 9 deletions libs/ngx-components/src/lib/components/audio/audio-viewer/audio-viewer.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ export class AudioViewerComponent implements OnInit, OnChanges, OnDestroy {
this.subscrManager = new SubscriptionManager<Subscription>();

this.subscrManager.add(
this.av.boundaryDragging.subscribe((status) => {
if (status === 'stopped') {
this.av.boundaryDragging.subscribe((event) => {
if (event.status === 'stopped') {
this.renderer.setStyle(
this.konvaContainer?.nativeElement,
'cursor',
Expand Down Expand Up @@ -253,7 +253,10 @@ export class AudioViewerComponent implements OnInit, OnChanges, OnDestroy {
* triggers when the boundary was dragged.
*/
@Output()
public get boundaryDragging(): Subject<'started' | 'stopped'> {
public get boundaryDragging(): Subject<{
status: 'started' | 'stopped' | 'dragging';
shiftPressed?: boolean;
}> {
return this.av.boundaryDragging;
}

Expand Down Expand Up @@ -1939,11 +1942,11 @@ export class AudioViewerComponent implements OnInit, OnChanges, OnDestroy {
(this.settings.lineheight + this.settings.margin.top);
const sceneSegment = this.av.currentLevel.items.find(
(a: any) => a.id === segment.id
) as OctraAnnotationSegment |undefined;
) as OctraAnnotationSegment | undefined;
if (
sceneSegment === undefined ||
segment?.time === undefined ||
(this.av.currentLevel.type !== AnnotationLevelType.SEGMENT)
this.av.currentLevel.type !== AnnotationLevelType.SEGMENT
) {
console.error(`scenceSegment is undefined!`);
} else {
Expand Down Expand Up @@ -2170,7 +2173,6 @@ export class AudioViewerComponent implements OnInit, OnChanges, OnDestroy {
this.audioChunk.absolutePlayposition.clone();
this.audioChunk.selection.end =
this.audioChunk.absolutePlayposition.clone();
this.av.drawnSelection = this.audioChunk.selection.clone();
}

this.av
Expand Down Expand Up @@ -2509,11 +2511,12 @@ export class AudioViewerComponent implements OnInit, OnChanges, OnDestroy {
private onKeyDown = (event: KeyboardEvent) => {
const shortcutInfo = this.shortcutsManager.checkKeyEvent(event, Date.now());

this.av.shiftPressed =
event.keyCode === 16 || event.code?.includes('Shift') || event.key?.includes('Shift');

if (shortcutInfo !== undefined) {
const comboKey = shortcutInfo.shortcut;

this.av.shiftPressed = comboKey === 'SHIFT';

if (this.settings.shortcutsEnabled) {
if (this._focused && this.isDisabledKey(comboKey)) {
// key pressed is disabled by config
Expand Down Expand Up @@ -2606,7 +2609,9 @@ export class AudioViewerComponent implements OnInit, OnChanges, OnDestroy {
if (
this.av.currentLevel.type === AnnotationLevelType.SEGMENT
) {
const segment = this.av.currentLevel.items[segmentI] as OctraAnnotationSegment<ASRContext>;
const segment = this.av.currentLevel.items[
segmentI
] as OctraAnnotationSegment<ASRContext>;
if (
segmentI > -1 &&
segment.context?.asr?.isBlockedBy === undefined &&
Expand Down Expand Up @@ -3118,6 +3123,7 @@ export class AudioViewerComponent implements OnInit, OnChanges, OnDestroy {
};

private onKeyUp = (event: KeyboardEvent) => {
this.av.onKeyUp();
this.shortcutsManager.checkKeyEvent(event, Date.now());
};

Expand Down
147 changes: 125 additions & 22 deletions libs/ngx-components/src/lib/components/audio/audio-viewer/audio-viewer.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
OctraAnnotation,
OctraAnnotationAnyLevel,
OctraAnnotationSegment,
OctraAnnotationSegmentLevel,
OLabel,
} from '@octra/annotation';
import { Subject } from 'rxjs';
Expand All @@ -27,7 +28,10 @@ import {
providedIn: 'root',
})
export class AudioViewerService {
get boundaryDragging(): Subject<'started' | 'stopped'> {
get boundaryDragging(): Subject<{
status: 'started' | 'stopped' | 'dragging';
shiftPressed?: boolean;
}> {
return this._boundaryDragging;
}

Expand Down Expand Up @@ -60,7 +64,10 @@ export class AudioViewerService {
protected mouseClickPos: SampleUnit | undefined;
protected playcursor: PlayCursor | undefined;

private _boundaryDragging: Subject<'started' | 'stopped'>;
private _boundaryDragging: Subject<{
status: 'started' | 'stopped' | 'dragging';
shiftPressed?: boolean;
}>;
currentLevelID?: number;

annotation?: OctraAnnotation<ASRContext, OctraAnnotationSegment>;
Expand Down Expand Up @@ -134,7 +141,10 @@ export class AudioViewerService {
set dragableBoundaryNumber(value: number) {
if (value > -1 && this._dragableBoundaryNumber === -1) {
// started
this._boundaryDragging.next('started');
this._boundaryDragging.next({
shiftPressed: this.shiftPressed,
status: 'started',
});
}
this._dragableBoundaryNumber = value;
}
Expand Down Expand Up @@ -185,7 +195,10 @@ export class AudioViewerService {
}

constructor(private multiThreadingService: MultiThreadingService) {
this._boundaryDragging = new Subject<'started' | 'stopped'>();
this._boundaryDragging = new Subject<{
status: 'started' | 'stopped' | 'dragging';
shiftPressed?: boolean;
}>();
}

public initialize(innerWidth: number, audioChunk: AudioChunk) {
Expand Down Expand Up @@ -233,7 +246,9 @@ export class AudioViewerService {
this.audioChunk.startpos = this.mouseClickPos.clone();
this.audioChunk.selection.start = absXInTime.clone();
this.audioChunk.selection.end = absXInTime.clone();
this._drawnSelection = this.audioChunk.selection.clone();
if (!this.shiftPressed) {
this._drawnSelection = this.audioChunk.selection.clone();
}

if (this._dragableBoundaryNumber > -1) {
const segmentBefore = (
Expand Down Expand Up @@ -286,23 +301,108 @@ export class AudioViewerService {
)?.clone();

if (segment) {
segment.time = this.audioTCalculator.absXChunktoSampleUnit(
absX,
this.audioChunk
)!;

this.currentLevelChange.emit({
type: 'change',
items: [
{
instance: segment,
},
],
});
this.annotationChange.emit(this.annotation);
if (!this.shiftPressed) {
// move only this boundary
segment.time = this.audioTCalculator.absXChunktoSampleUnit(
absX,
this.audioChunk
)!;

this.currentLevelChange.emit({
type: 'change',
items: [
{
instance: segment,
},
],
});
this.annotationChange.emit(this.annotation);
} else if (this.drawnSelection?.duration?.samples) {
// move all segments with difference to left or right
const oldSamplePosition = segment.time.samples;
const newSamplePosition =
this.audioTCalculator.absXChunktoSampleUnit(
absX,
this.audioChunk
)?.samples;
const diff = newSamplePosition! - oldSamplePosition;
let changedItems: OctraAnnotationSegment[] = [];

if (diff > 0) {
// shift to right
for (const currentLevelElement of (this.annotation
.currentLevel as OctraAnnotationSegmentLevel<OctraAnnotationSegment>)!
.items) {
if (
currentLevelElement.time.samples >=
segment.time.samples &&
currentLevelElement.time.samples + diff <
this.drawnSelection.end!.samples
) {
const newItem = currentLevelElement.clone(
currentLevelElement.id
);
newItem.time = currentLevelElement.time.add(
this.audioManager.createSampleUnit(diff)
);
this.annotation =
this.annotation.changeCurrentItemById(
currentLevelElement.id,
newItem
);
changedItems.push(newItem);
}
}
} else {
// shift to left
for (const currentLevelElement of (this.annotation
.currentLevel as OctraAnnotationSegmentLevel<OctraAnnotationSegment>)!
.items) {
console.log(
`move to ${currentLevelElement.time.samples + diff}`
);
if (
currentLevelElement.time.samples <=
segment.time.samples &&
currentLevelElement.time.samples + diff >
this.drawnSelection.start!.samples
) {
const newItem = currentLevelElement.clone(
currentLevelElement.id
);
newItem.time = currentLevelElement.time.add(
this.audioManager.createSampleUnit(diff)
);
this.annotation =
this.annotation.changeCurrentItemById(
currentLevelElement.id,
newItem
);
changedItems.push(newItem);
} else if (
currentLevelElement.time.samples - diff <
0
) {
changedItems = [];
break;
}
}
}

if (changedItems.length > 0) {
this.currentLevelChange.emit({
type: 'change',
items: changedItems.map((a) => ({ instance: a })),
});
this.annotationChange.emit(this.annotation);
}
}
}

this._boundaryDragging.next('stopped');
this._boundaryDragging.next({
shiftPressed: this.shiftPressed,
status: 'stopped',
});
} else {
// set selection
this.audioChunk.selection.end = absXInTime.clone();
Expand Down Expand Up @@ -612,8 +712,11 @@ export class AudioViewerService {

if (this.mouseDown && this._dragableBoundaryNumber < 0) {
// mouse down, nothing dragged
this.audioChunk.selection.end = absXTime.clone();
this._drawnSelection = this.audioChunk.selection.clone();
if (!this.shiftPressed) {
console.log('reset selection');
this.audioChunk.selection.end = absXTime.clone();
this._drawnSelection = this.audioChunk.selection.clone();
}
} else if (
this.settings.boundaries.enabled &&
this.mouseDown &&
Expand Down

0 comments on commit 0af071e

Please sign in to comment.