From 3238ca2c971af66c8beaf713eb2e569cf85bfc02 Mon Sep 17 00:00:00 2001
From: Roman <roman.buchholz@t-online.de>
Date: Thu, 2 Jul 2020 18:54:47 +0200
Subject: [PATCH] discretize output and textual explanations

---
 frontend/src/App.tsx                   | 19 ++++++++++++++-----
 frontend/src/ExplanationsContainer.tsx | 22 ++++++++++++++++++----
 frontend/src/Participant.tsx           |  7 +++++--
 frontend/src/loadEngagementData.ts     |  8 ++++----
 4 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index a7a9534..952d4a0 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -43,6 +43,7 @@ type StateType = {
     sessionId: string;
     mode: 'bar' | 'cloud';
     participantsData: ParticipantData[],
+    discreteParticipantsData: ParticipantData[],
     loading: boolean,
     error: Error | null,
     paused: boolean,
@@ -58,6 +59,7 @@ class App extends React.Component<{}, StateType> {
         mode: "bar",
         loading: false,
         participantsData: [],
+        discreteParticipantsData: [],
         error: null,
         paused: true,
         showDevSettings: true,
@@ -101,15 +103,19 @@ class App extends React.Component<{}, StateType> {
         this.setState({ loading: true });
         try {
             const participantsData = this.getEmptyParticipantsData();
-            for (let pData of participantsData) {
-                pData.dataContainer = await loadEngagementData(
+            const discretePartData = this.getEmptyParticipantsData();
+            for (let i = 0; i < participantsData.length; i++) {
+                let engagementData = await loadEngagementData(
                     this.state.username,
                     this.state.password,
-                    pData.dataURL,
+                    participantsData[i].dataURL,
                     false
                 );
+                participantsData[i].dataContainer = engagementData[0];
+                discretePartData[i].dataContainer = engagementData[1];
             }
             this.setState({ participantsData });
+            this.setState({ discreteParticipantsData: discretePartData });
         } catch (error) {
             this.setState({ error });
         } finally {
@@ -125,8 +131,11 @@ class App extends React.Component<{}, StateType> {
 
     renderParticipants = () => {
         return this.state.participantsData
-            .map((item, index) => item.dataContainer !== null
-                ? <Participant key={"p" + index} dataContainer={item.dataContainer} name={item.name} gender={item.gender}
+            .map((item, index) =>
+                item.dataContainer !== null && this.state.discreteParticipantsData[index].dataContainer !== null
+                ? <Participant key={"p" + index} dataContainer={item.dataContainer}
+                    discreteContainer={this.state.discreteParticipantsData[index].dataContainer as DataContainerType }
+                    name={item.name} gender={item.gender}
                     volume={this.state.volume}
                     videoURL={item.videoURL} mode={this.state.mode} paused={this.state.paused} />
                 : null)
diff --git a/frontend/src/ExplanationsContainer.tsx b/frontend/src/ExplanationsContainer.tsx
index 58de68c..fb5c004 100644
--- a/frontend/src/ExplanationsContainer.tsx
+++ b/frontend/src/ExplanationsContainer.tsx
@@ -70,6 +70,7 @@ const Unsure = styled.div`
 
 function ExplanationsContainer(props: {
     dataPoint: { input: number[]; output: number[]; explanations: number[][] } | null;
+    discreteDataPoint: { input: number[]; output: number[]; explanations: number[][] } | null;
     labels: string[];
     maxExplanationValue: number;
     minInputValues: number[];
@@ -78,13 +79,17 @@ function ExplanationsContainer(props: {
     username: string;
     gender: Gender;
 }) {
-    const { maxInputValues, minInputValues, maxExplanationValue, dataPoint, username, gender, labels, mode } = props;
+    const { maxInputValues, minInputValues, maxExplanationValue, dataPoint, discreteDataPoint, username, gender,
+        labels, mode } = props;
     if (!dataPoint) return <Container />;
+    if (!discreteDataPoint) return <Container />;
     const { input, output, explanations } = dataPoint;
     const strongestOutputIdx = output.indexOf(Math.max(...output));
+    const discreteStrongestOutputIdx = discreteDataPoint.output.indexOf(Math.max(...discreteDataPoint.output));
     const confidence = Math.round(output[strongestOutputIdx] * 1000) / 10;
 
     const inputNormalized = normalizeInput(input, minInputValues, maxInputValues);
+    const discreteInputNormalized = normalizeInput(discreteDataPoint.input, minInputValues, maxInputValues);
 
     const strongestOutputExplanations = sortAndSelectTopmostFeatures(
         labels,
@@ -95,18 +100,27 @@ function ExplanationsContainer(props: {
         true
     );
 
+    const discreteStrongestOutputExplanations = sortAndSelectTopmostFeatures(
+        labels,
+        discreteInputNormalized,
+        discreteDataPoint.explanations[discreteStrongestOutputIdx],
+        3,
+        0.2,
+        true
+    );
+
     const blur = calculateBlur(confidence);
     const colorPalette = strongestOutputIdx < 2 ? ENGAGEMENT_NEGATIVE_COLOR_PALETTE : ENGAGEMENT_POSITIVE_COLOR_PALETTE;
 
     return (
         <Container>
             <Explanations style={{ filter: `blur(${blur}px)` }}>
-                <ExplanationsHeading strongestOutputIdx={strongestOutputIdx} />
+                <ExplanationsHeading strongestOutputIdx={discreteStrongestOutputIdx} />
                 <BasedOn>
                     <span>Based on: </span>
                     <FeatureActivationTextDescription
-                        categoryIds={strongestOutputExplanations.topMostLabels}
-                        categoryValues={strongestOutputExplanations.topMostInputs}
+                        categoryIds={discreteStrongestOutputExplanations.topMostLabels}
+                        categoryValues={discreteStrongestOutputExplanations.topMostInputs}
                         username={username}
                         userGender={gender}
                         popOverDisabled={blur > 0}
diff --git a/frontend/src/Participant.tsx b/frontend/src/Participant.tsx
index d3a724f..4f28a50 100644
--- a/frontend/src/Participant.tsx
+++ b/frontend/src/Participant.tsx
@@ -31,6 +31,7 @@ class Participant extends React.Component<{
     videoURL: string; name: string;
         gender: Gender;
     dataContainer: DataContainerType;
+    discreteContainer: DataContainerType;
     mode: "bar" | "cloud";
     paused: boolean,
     volume: number
@@ -43,11 +44,12 @@ class Participant extends React.Component<{
     };
 
     render() {
-        const {videoURL, name, dataContainer, mode, volume} = this.props;
+        const {videoURL, name, dataContainer, discreteContainer, mode, volume} = this.props;
         const {currentTime} = this.state;
         const dataPoint = dataContainer?.data[Math.floor(currentTime * dataContainer?.sampleRate)];
+        const discreteDataPoint = discreteContainer?.data[Math.floor(currentTime * dataContainer?.sampleRate)];
 
-        const outputClass = dataPoint ? dataPoint.output.indexOf(Math.max(...dataPoint.output)) : 4;
+        const outputClass = discreteDataPoint ? discreteDataPoint.output.indexOf(Math.max(...discreteDataPoint.output)) : 4;
 
         return (
             <CustomCard elevation={Elevation.TWO}>
@@ -61,6 +63,7 @@ class Participant extends React.Component<{
                 <ExplanationsContainer
                     labels={dataContainer?.labels || []}
                     dataPoint={dataPoint}
+                    discreteDataPoint={discreteDataPoint}
                     mode={mode}
                     username={name}
                     gender={this.props.gender}
diff --git a/frontend/src/loadEngagementData.ts b/frontend/src/loadEngagementData.ts
index 4e0dc6b..dd21c8b 100644
--- a/frontend/src/loadEngagementData.ts
+++ b/frontend/src/loadEngagementData.ts
@@ -230,15 +230,15 @@ const loadEngagementData = async (
         dataContainer.data = smoothData(dataContainer.data, windowSize);
     }
 
-    if (discretizeValues) {
-        dataContainer.data = discretizeDataPoints(dataContainer.data, dataContainer.sampleRate * 2, discretizeOutputOnly);
-    }
+    let discreteDataContainer: DataContainerType = JSON.parse(JSON.stringify(dataContainer));
+    discreteDataContainer.data = discretizeDataPoints(discreteDataContainer.data, dataContainer.sampleRate * 3, false);
+    dataContainer.data = discretizeDataPoints(dataContainer.data, dataContainer.sampleRate * 3, true);
 
     let maxValues = maxExplanationsAndMinMaxInputValue(dataContainer.data);
     dataContainer.maxExplanationValue = maxValues.maxExplanation;
     dataContainer.maxInputs = maxValues.maxInputValues;
     dataContainer.minInputs = maxValues.minInputValues;
-    return categorize(dataContainer, featuresToCategoryMapping);
+    return [categorize(dataContainer, featuresToCategoryMapping), categorize(discreteDataContainer, featuresToCategoryMapping)];
 };
 
 const smoothUsingPredictions = (dataContainer: DataContainerType, windowSize: number): DataPointType[] => {