From ed3a6c7480be5b248607835722a5e027290a3139 Mon Sep 17 00:00:00 2001 From: Jarred Heinrich Date: Tue, 31 Jan 2017 12:55:55 -0800 Subject: [PATCH 1/4] added motion mask partner filter, first pass --- .../suripu/core/flipper/FeatureFlipper.java | 1 + .../processors/FeatureFlippedProcessor.java | 3 + .../com/hello/suripu/core/util/CSVLoader.java | 39 +++++--- .../core/util/MotionMaskPartnerFilter.java | 92 +++++++++++++++++++ .../util/MotionMaskPartnerFilterTest.java | 20 ++++ .../fixtures/tracker_motion/mm_pill_data1.csv | 75 +++++++++++++++ .../fixtures/tracker_motion/mm_pill_data2.csv | 29 ++++++ .../InstrumentedTimelineProcessor.java | 10 ++ 8 files changed, 258 insertions(+), 11 deletions(-) create mode 100644 suripu-core/src/main/java/com/hello/suripu/core/util/MotionMaskPartnerFilter.java create mode 100644 suripu-core/src/test/java/com/hello/suripu/core/util/MotionMaskPartnerFilterTest.java create mode 100644 suripu-core/src/test/resources/fixtures/tracker_motion/mm_pill_data1.csv create mode 100644 suripu-core/src/test/resources/fixtures/tracker_motion/mm_pill_data2.csv diff --git a/suripu-core/src/main/java/com/hello/suripu/core/flipper/FeatureFlipper.java b/suripu-core/src/main/java/com/hello/suripu/core/flipper/FeatureFlipper.java index 4bf24fd11..f047c6a0f 100644 --- a/suripu-core/src/main/java/com/hello/suripu/core/flipper/FeatureFlipper.java +++ b/suripu-core/src/main/java/com/hello/suripu/core/flipper/FeatureFlipper.java @@ -59,6 +59,7 @@ public class FeatureFlipper { public final static String MEASURE_CLOCK_DRIFT = "measure_clock_drift"; public final static String MISSING_DATA_DEFAULT_VALUE = "missing_data_default_value"; + public final static String MOTION_MASK_PARTNER_FILTER = "motion_mask_partner_filter"; public final static String NEW_INVALID_NIGHT_FILTER = "new_invalid_night_filter"; diff --git a/suripu-core/src/main/java/com/hello/suripu/core/processors/FeatureFlippedProcessor.java b/suripu-core/src/main/java/com/hello/suripu/core/processors/FeatureFlippedProcessor.java index 84954b405..f542c63c4 100644 --- a/suripu-core/src/main/java/com/hello/suripu/core/processors/FeatureFlippedProcessor.java +++ b/suripu-core/src/main/java/com/hello/suripu/core/processors/FeatureFlippedProcessor.java @@ -191,4 +191,7 @@ protected Boolean useUninterruptedDuration(final Long accountId) { protected Boolean useSmartAlarmRefactored(final Long accountId){ return featureFlipper.userFeatureActive(FeatureFlipper.SMART_ALARM_REFACTORED, accountId, Collections.EMPTY_LIST); } + protected Boolean hasMotionMaskPartnerFilter(final Long accountId){ + return featureFlipper.userFeatureActive(FeatureFlipper.MOTION_MASK_PARTNER_FILTER, accountId, Collections.EMPTY_LIST); + } } diff --git a/suripu-core/src/main/java/com/hello/suripu/core/util/CSVLoader.java b/suripu-core/src/main/java/com/hello/suripu/core/util/CSVLoader.java index 853083e89..a9638660a 100644 --- a/suripu-core/src/main/java/com/hello/suripu/core/util/CSVLoader.java +++ b/suripu-core/src/main/java/com/hello/suripu/core/util/CSVLoader.java @@ -23,17 +23,34 @@ public static List loadTrackerMotionFromCSV(final String resource final String[] lines = csvString.split("\\n"); for(int i = 1; i < lines.length; i++){ final String[] columns = lines[i].split(","); - final TrackerMotion trackerMotion = new TrackerMotion( - Long.parseLong(columns[0].trim()), //id - Long.parseLong(columns[1].trim()), //account id - 0L, // tracker_id - Long.parseLong(columns[3].trim()), //timestamp - Integer.parseInt(columns[4].trim()), //value - Integer.parseInt(columns[5].trim()), // offset - Long.parseLong(columns[6].trim()), //motion_range - Long.parseLong(columns[7].trim()), //kickoff count - Long.parseLong(columns[8].trim()) //on duration - ); + final TrackerMotion trackerMotion; + if (columns.length >=11) { + final TrackerMotion.Builder trackerMotionBuilder = new TrackerMotion.Builder(); + trackerMotion = trackerMotionBuilder.withAccountId(Long.parseLong(columns[1].trim())) + .withExternalTrackerId("0") + .withTimestampMillis(Long.parseLong(columns[3].trim())) //timestamp + .withValue(Integer.parseInt(columns[4].trim())) //value + .withOffsetMillis(Integer.parseInt(columns[5].trim())) // offset + .withMotionRange(Long.parseLong(columns[6].trim())) //motion_range + .withKickOffCounts(Long.parseLong(columns[7].trim())) //kickoff count + .withOnDurationInSeconds(Long.parseLong(columns[8].trim())) //on duration + .withMotionMask(Long.parseLong(columns[9].trim())) + .withCosTheta(Long.parseLong(columns[10].trim())) + .build(); + }else{ + trackerMotion = new TrackerMotion( + Long.parseLong(columns[0].trim()), //id + Long.parseLong(columns[1].trim()), //account id + 0L, // tracker_id + Long.parseLong(columns[3].trim()), //timestamp + Integer.parseInt(columns[4].trim()), //value + Integer.parseInt(columns[5].trim()), // offset + Long.parseLong(columns[6].trim()), //motion_range + Long.parseLong(columns[7].trim()), //kickoff count + Long.parseLong(columns[8].trim()) //on duration + ); + + } trackerMotions.add(trackerMotion); //} } diff --git a/suripu-core/src/main/java/com/hello/suripu/core/util/MotionMaskPartnerFilter.java b/suripu-core/src/main/java/com/hello/suripu/core/util/MotionMaskPartnerFilter.java new file mode 100644 index 000000000..ea59a69ee --- /dev/null +++ b/suripu-core/src/main/java/com/hello/suripu/core/util/MotionMaskPartnerFilter.java @@ -0,0 +1,92 @@ +package com.hello.suripu.core.util; + +import com.hello.suripu.core.models.TrackerMotion; +import com.hello.suripu.core.models.motion.PartnerMotionAtSecond; +import com.hello.suripu.core.models.motion.PartnerMotionTimeSeries; +import com.hello.suripu.core.models.motion.TrackerMotionWithPartnerMotion; +import org.joda.time.DateTimeConstants; + +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; + +/** + * Created by jarredheinrich on 12/28/16. + */ +public class MotionMaskPartnerFilter { + final private static int UNCERTAINTY_SECS = 1; + //add uncertainty + public static List partnerFiltering(final List originalTrackerMotions, + final List originalPartnerMotions){ + + if (originalTrackerMotions.isEmpty() || originalPartnerMotions.isEmpty()){ + return originalTrackerMotions; + } + //requires motion mask + if(!originalTrackerMotions.get(0).motionMask.isPresent() ||!originalPartnerMotions.get(0).motionMask.isPresent()){ + return originalTrackerMotions; + } + + final PartnerMotionTimeSeries originalMotionTimeSeries= PartnerMotionTimeSeries.create(originalTrackerMotions,originalPartnerMotions); + final List originalTrackerMotionTimeSeriesWithPartner = originalMotionTimeSeries.groupByLeft(); + final List trackerMotions = new ArrayList<>(); + final List removedTrackerMotionsWithPartnerMotion = new ArrayList<>(); + + + PartnerMotionAtSecond motionAtSecond; + + final ListIterator trackerMotionIterator = originalTrackerMotionTimeSeriesWithPartner.listIterator(); + TrackerMotionWithPartnerMotion previousTrackerMotionWithPartnerMotion = originalTrackerMotionTimeSeriesWithPartner.get(0); + for (final TrackerMotionWithPartnerMotion trackerMotionWithPartner : originalTrackerMotionTimeSeriesWithPartner){ + boolean userMovement = false; + boolean partnerMovedAtSecond = false; + if (trackerMotionIterator.hasPrevious()){ + final int interMotionTimeDiffSeconds =(int) (trackerMotionWithPartner.trackerMotion.timestamp - previousTrackerMotionWithPartnerMotion.trackerMotion.timestamp )/ DateTimeConstants.MILLIS_PER_SECOND; + if (interMotionTimeDiffSeconds <= 60 && interMotionTimeDiffSeconds > 0) { + partnerMovedAtSecond= previousTrackerMotionWithPartnerMotion.partnerMotionAtSeconds.get(interMotionTimeDiffSeconds - 1).didPartnerMove; + } + } + PartnerMotionAtSecond motionAtNextSecond = trackerMotionWithPartner.partnerMotionAtSeconds.get(0); + boolean partnerMovedAtNextSecond = motionAtNextSecond.didPartnerMove; + + + for (int second = 0; second < 60; second ++){ + motionAtSecond = motionAtNextSecond; + if (trackerMotionWithPartner.partnerMotionAtSeconds.size() > second + UNCERTAINTY_SECS){ + motionAtNextSecond = trackerMotionWithPartner.partnerMotionAtSeconds.get(second + UNCERTAINTY_SECS); + partnerMovedAtNextSecond = motionAtNextSecond.didPartnerMove; + }else if(trackerMotionIterator.hasNext()){ + TrackerMotionWithPartnerMotion nextTrackerMotionWithPartner = trackerMotionIterator.next(); + final int interMotionTimeDiffSeconds =(int) (nextTrackerMotionWithPartner.trackerMotion.timestamp - trackerMotionWithPartner.trackerMotion.timestamp)/ DateTimeConstants.MILLIS_PER_SECOND; + if (interMotionTimeDiffSeconds <= 60 && interMotionTimeDiffSeconds > 0) { + motionAtNextSecond = nextTrackerMotionWithPartner.partnerMotionAtSeconds.get(DateTimeConstants.SECONDS_PER_MINUTE - interMotionTimeDiffSeconds); + partnerMovedAtNextSecond = motionAtNextSecond.didPartnerMove; + } + } + Boolean partnerMovedAtPrevSecond = partnerMovedAtSecond; + partnerMovedAtSecond = motionAtSecond.didPartnerMove; + + //did user move + if(!motionAtSecond.didMove){ + continue; + } + //did partnerMove + if (partnerMovedAtNextSecond || partnerMovedAtSecond || partnerMovedAtPrevSecond){ + continue; + } + + userMovement = true; + } + + + if (userMovement){ + trackerMotions.add(trackerMotionWithPartner.trackerMotion); + }else{ + removedTrackerMotionsWithPartnerMotion.add(trackerMotionWithPartner); + } + previousTrackerMotionWithPartnerMotion = trackerMotionWithPartner; + + } + return trackerMotions; + } +} diff --git a/suripu-core/src/test/java/com/hello/suripu/core/util/MotionMaskPartnerFilterTest.java b/suripu-core/src/test/java/com/hello/suripu/core/util/MotionMaskPartnerFilterTest.java new file mode 100644 index 000000000..fb3e1b0d4 --- /dev/null +++ b/suripu-core/src/test/java/com/hello/suripu/core/util/MotionMaskPartnerFilterTest.java @@ -0,0 +1,20 @@ +package com.hello.suripu.core.util; + +import com.hello.suripu.core.models.TrackerMotion; +import org.junit.Test; + +import java.util.List; + +/** + * Created by jarredheinrich on 1/30/17. + */ +public class MotionMaskPartnerFilterTest { + @Test + public void testPartnerFiltering() { + final List trackerMotions = CSVLoader.loadTrackerMotionFromCSV("fixtures/tracker_motion/mm_pill_data1.csv"); + final List partnerTrackerMotions = CSVLoader.loadTrackerMotionFromCSV("fixtures/tracker_motion/mm_pill_data2.csv"); + final List filteredTrackerMotions = MotionMaskPartnerFilter.partnerFiltering(trackerMotions, partnerTrackerMotions); + assert(filteredTrackerMotions.size() == 72); + } + +} diff --git a/suripu-core/src/test/resources/fixtures/tracker_motion/mm_pill_data1.csv b/suripu-core/src/test/resources/fixtures/tracker_motion/mm_pill_data1.csv new file mode 100644 index 000000000..091c473db --- /dev/null +++ b/suripu-core/src/test/resources/fixtures/tracker_motion/mm_pill_data1.csv @@ -0,0 +1,75 @@ +,account_id,device_id,ts,val,offset,motion_range,kickoff_count,onDur,motion_mask,cost +0,66376,F832678F8B4E1FEC,1485748888000,9350,-28800000,0,0,4,204472320,0 +1,66376,F832678F8B4E1FEC,1485749308000,2758,-28800000,0,0,8,2139095040,0 +2,66376,F832678F8B4E1FEC,1485754707000,20540,-28800000,0,0,9,1134937892422942720,0 +3,66376,F832678F8B4E1FEC,1485754767000,20692,-28800000,0,0,11,72057594037928959,122 +4,66376,F832678F8B4E1FEC,1485754827000,2452,-28800000,0,0,9,547609378816,129 +5,66376,F832678F8B4E1FEC,1485755008000,0,-28800000,0,0,3,109051904,0 +6,66376,F832678F8B4E1FEC,1485755128000,-154,-28800000,0,0,1,4398046511104,0 +7,66376,F832678F8B4E1FEC,1485755668000,0,-28800000,0,0,2,12884901888,0 +8,66376,F832678F8B4E1FEC,1485756508000,306,-28800000,0,0,4,1125912791875584,0 +9,66376,F832678F8B4E1FEC,1485756988000,-308,-28800000,0,0,1,8388608,0 +10,66376,F832678F8B4E1FEC,1485757048000,0,-28800000,0,0,4,30399297484750848,131 +11,66376,F832678F8B4E1FEC,1485757108000,612,-28800000,0,0,4,67553994410557440,131 +12,66376,F832678F8B4E1FEC,1485757168000,-308,-28800000,0,0,1,140737488355328,131 +13,66376,F832678F8B4E1FEC,1485757648000,-154,-28800000,0,0,2,805306368,0 +14,66376,F832678F8B4E1FEC,1485757708000,152,-28800000,0,0,3,83968,131 +15,66376,F832678F8B4E1FEC,1485758668000,306,-28800000,0,0,2,196608,0 +16,66376,F832678F8B4E1FEC,1485759028000,-154,-28800000,0,0,2,68719476864,0 +17,66376,F832678F8B4E1FEC,1485759088000,-154,-28800000,0,0,1,4096,131 +18,66376,F832678F8B4E1FEC,1485760108000,-154,-28800000,0,0,3,23068672,0 +19,66376,F832678F8B4E1FEC,1485760648000,-154,-28800000,0,0,6,435582526459740160,0 +20,66376,F832678F8B4E1FEC,1485760708000,1378,-28800000,0,0,4,25904021505,124 +21,66376,F832678F8B4E1FEC,1485761008000,0,-28800000,0,0,1,68719476736,0 +22,66376,F832678F8B4E1FEC,1485761608000,0,-28800000,0,0,1,65536,0 +23,66376,F832678F8B4E1FEC,1485761668000,306,-28800000,0,0,2,3298534883328,124 +24,66376,F832678F8B4E1FEC,1485761728000,152,-28800000,0,0,2,100663296,124 +25,66376,F832678F8B4E1FEC,1485761788000,920,-28800000,0,0,4,13052674048,124 +26,66376,F832678F8B4E1FEC,1485763588000,-154,-28800000,0,0,2,3377699720527872,0 +27,66376,F832678F8B4E1FEC,1485763708000,-154,-28800000,0,0,4,19058917376,0 +28,66376,F832678F8B4E1FEC,1485764128000,460,-28800000,0,0,2,216172782113783808,0 +29,66376,F832678F8B4E1FEC,1485764248000,-308,-28800000,0,0,1,274877906944,0 +30,66376,F832678F8B4E1FEC,1485764308000,-154,-28800000,0,0,2,12582912,124 +31,66376,F832678F8B4E1FEC,1485764668000,152,-28800000,0,0,3,450359962737049600,0 +32,66376,F832678F8B4E1FEC,1485764728000,0,-28800000,0,0,3,28,124 +33,66376,F832678F8B4E1FEC,1485764968000,-154,-28800000,0,0,1,70368744177664,0 +34,66376,F832678F8B4E1FEC,1485765448000,-154,-28800000,0,0,2,2164260864,0 +35,66376,F832678F8B4E1FEC,1485765508000,5670,-28800000,0,0,8,66228395704320,124 +36,66376,F832678F8B4E1FEC,1485765628000,0,-28800000,0,0,2,824633720832,0 +37,66376,F832678F8B4E1FEC,1485767728000,306,-28800000,0,0,9,518782976,0 +38,66376,F832678F8B4E1FEC,1485767908000,766,-28800000,0,0,11,3071,131 +39,66376,F832678F8B4E1FEC,1485768088000,612,-28800000,0,0,6,63,0 +40,66376,F832678F8B4E1FEC,1485768508000,0,-28800000,0,0,2,603979776,0 +41,66376,F832678F8B4E1FEC,1485768868000,460,-28800000,0,0,8,13941866496,0 +42,66376,F832678F8B4E1FEC,1485769228000,152,-28800000,0,0,4,14918173765664768,0 +43,66376,F832678F8B4E1FEC,1485770368000,0,-28800000,0,0,3,360777252864,0 +44,66376,F832678F8B4E1FEC,1485770788000,2298,-28800000,0,0,10,559754497753088,0 +45,66376,F832678F8B4E1FEC,1485772408000,-154,-28800000,0,0,1,2048,0 +46,66376,F832678F8B4E1FEC,1485773788000,-154,-28800000,0,0,4,159877786771652608,0 +47,66376,F832678F8B4E1FEC,1485773848000,-154,-28800000,0,0,1,288230376151711744,131 +48,66376,F832678F8B4E1FEC,1485773908000,-154,-28800000,0,0,3,521,124 +49,66376,F832678F8B4E1FEC,1485774928000,-154,-28800000,0,0,2,105553116266496,0 +50,66376,F832678F8B4E1FEC,1485775468000,-154,-28800000,0,0,2,98304,0 +51,66376,F832678F8B4E1FEC,1485775708000,-308,-28800000,0,0,1,18014398509481984,0 +52,66376,F832678F8B4E1FEC,1485776908000,306,-28800000,0,0,5,403726925824,0 +53,66376,F832678F8B4E1FEC,1485778228000,1072,-28800000,0,0,6,17460244649082880,0 +54,66376,F832678F8B4E1FEC,1485778828000,-154,-28800000,0,0,2,19791209299968,0 +55,66376,F832678F8B4E1FEC,1485779488000,-308,-28800000,0,0,1,17179869184,0 +56,66376,F832678F8B4E1FEC,1485781647000,-308,-28800000,0,0,1,17592186044416,0 +57,66376,F832678F8B4E1FEC,1485782007000,-308,-28800000,0,0,1,512,0 +58,66376,F832678F8B4E1FEC,1485782127000,-154,-28800000,0,0,1,35184372088832,0 +59,66376,F832678F8B4E1FEC,1485786387000,152,-28800000,0,0,6,750975237864030208,0 +60,66376,F832678F8B4E1FEC,1485786447000,1072,-28800000,0,0,3,7,131 +61,66376,F832678F8B4E1FEC,1485786507000,-154,-28800000,0,0,2,192,131 +62,66376,F832678F8B4E1FEC,1485786867000,920,-28800000,0,0,6,8373880557142016,131 +63,66376,F832678F8B4E1FEC,1485787048000,2452,-28800000,0,0,7,35747322042253312,0 +64,66376,F832678F8B4E1FEC,1485787228000,-154,-28800000,0,0,1,17592186044416,0 +65,66376,F832678F8B4E1FEC,1485787288000,766,-28800000,0,0,7,16189243567112192,131 +66,66376,F832678F8B4E1FEC,1485787348000,152,-28800000,0,0,1,4,131 +67,66376,F832678F8B4E1FEC,1485787408000,152,-28800000,0,0,3,1792,131 +68,66376,F832678F8B4E1FEC,1485787947000,1992,-28800000,0,0,3,58720256,0 +69,66376,F832678F8B4E1FEC,1485788367000,306,-28800000,0,0,3,2751463424,0 +70,66376,F832678F8B4E1FEC,1485788847000,152,-28800000,0,0,2,216172782113783808,0 +71,66376,F832678F8B4E1FEC,1485788907000,9810,-28800000,0,0,4,36028822788784128,131 +72,66376,F832678F8B4E1FEC,1485788967000,9656,-28800000,0,0,9,8796205219868,124 +73,66376,F832678F8B4E1FEC,1485789027000,15788,-28800000,0,0,15,6755657403334700,125 diff --git a/suripu-core/src/test/resources/fixtures/tracker_motion/mm_pill_data2.csv b/suripu-core/src/test/resources/fixtures/tracker_motion/mm_pill_data2.csv new file mode 100644 index 000000000..96a5e2eed --- /dev/null +++ b/suripu-core/src/test/resources/fixtures/tracker_motion/mm_pill_data2.csv @@ -0,0 +1,29 @@ +,account_id,device_id,ts,val,offset,motion_range,kickoff_count,onDur,motion_mask,cost +0,66377,DE77624F2DD30592,1485749008000,0,-28800000,0,0,1,33554432,0 +1,66377,DE77624F2DD30592,1485749068000,306,-28800000,0,0,4,201394176,131 +2,66377,DE77624F2DD30592,1485749128000,152,-28800000,0,0,2,4503599761588224,131 +3,66377,DE77624F2DD30592,1485749248000,152,-28800000,0,0,1,1048576,0 +4,66377,DE77624F2DD30592,1485754708000,152,-28800000,0,0,2,72066390130950144,0 +5,66377,DE77624F2DD30592,1485754768000,21458,-28800000,0,0,6,126100789566373902,131 +6,66377,DE77624F2DD30592,1485754828000,27744,-28800000,0,0,16,3377837059735552,130 +7,66377,DE77624F2DD30592,1485754888000,0,-28800000,0,0,4,70420283785280,130 +8,66377,DE77624F2DD30592,1485754949000,0,-28800000,0,0,2,549755813889,131 +9,66377,DE77624F2DD30592,1485755309000,-154,-28800000,0,0,1,512,0 +10,66377,DE77624F2DD30592,1485764728000,3372,-28800000,0,0,6,63,131 +11,66377,DE77624F2DD30592,1485765509000,306,-28800000,0,0,1,34359738368,131 +12,66377,DE77624F2DD30592,1485765629000,152,-28800000,0,0,4,1700807049216,0 +13,66377,DE77624F2DD30592,1485768509000,0,-28800000,0,0,1,1048576,0 +14,66377,DE77624F2DD30592,1485769949000,-154,-28800000,0,0,1,16384,0 +15,66377,DE77624F2DD30592,1485770189000,0,-28800000,0,0,1,8,0 +16,66377,DE77624F2DD30592,1485770788000,306,-28800000,0,0,4,263882790666240,0 +17,66377,DE77624F2DD30592,1485771328000,0,-28800000,0,0,2,13194139533312,0 +18,66377,DE77624F2DD30592,1485774748000,152,-28800000,0,0,2,6597069766656,0 +19,66377,DE77624F2DD30592,1485776068000,0,-28800000,0,0,1,16,0 +20,66377,DE77624F2DD30592,1485778709000,0,-28800000,0,0,3,3848290697216,0 +21,66377,DE77624F2DD30592,1485782728000,-154,-28800000,0,0,1,8,0 +22,66377,DE77624F2DD30592,1485784288000,306,-28800000,0,0,4,540431955284459520,0 +23,66377,DE77624F2DD30592,1485785667000,0,-28800000,0,0,1,4,0 +24,66377,DE77624F2DD30592,1485786747000,0,-28800000,0,0,2,3298534883328,0 +25,66377,DE77624F2DD30592,1485786807000,-154,-28800000,0,0,4,35700736,131 +26,66377,DE77624F2DD30592,1485789028000,3218,-28800000,0,0,16,1030993588691,130 +27,66377,DE77624F2DD30592,1485789208000,0,-28800000,0,0,1,34359738368,0 diff --git a/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java b/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java index f88ccca02..c1eabc9fa 100644 --- a/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java +++ b/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java @@ -62,6 +62,7 @@ import com.hello.suripu.core.util.DateTimeUtil; import com.hello.suripu.core.util.FeedbackUtils; import com.hello.suripu.core.util.InBedSearcher; +import com.hello.suripu.core.util.MotionMaskPartnerFilter; import com.hello.suripu.core.util.OutlierFilter; import com.hello.suripu.core.util.PartnerDataUtils; import com.hello.suripu.core.util.SleepScoreUtils; @@ -513,6 +514,10 @@ protected Optional getSensorData(final long accountId,final D filteredOriginalMotions = OutlierFilter.removeOutliers(originalTrackerMotions, OUTLIER_GUARD_DURATION, DOMINANT_GROUP_DURATION); filteredOriginalPartnerMotions = OutlierFilter.removeOutliers(originalPartnerMotions, OUTLIER_GUARD_DURATION, DOMINANT_GROUP_DURATION); } + if (filteredOriginalMotions.isEmpty()){ + LOGGER.warn("action=outlier-filter-removed-all-motion account_id={}", accountId); + return Optional.absent(); + } final List trackerMotions = Lists.newArrayList(); @@ -530,6 +535,10 @@ protected Optional getSensorData(final long accountId,final D trackerMotions.addAll(filteredOriginalMotions); } } + else if (this.hasMotionMaskPartnerFilter(accountId) && originalTrackerMotions.get(0).motionMask.isPresent()){ + trackerMotions.addAll(MotionMaskPartnerFilter.partnerFiltering(filteredOriginalMotions, filteredOriginalPartnerMotions)); + LOGGER.info("action=using-motion-mask-partner-filter account_id={} removed-motions={}", accountId, filteredOriginalMotions.size() - trackerMotions.size()); + } else if (this.hasHmmPartnerFilterEnabled(accountId)) { LOGGER.info("using hmm partner filter"); try { @@ -548,6 +557,7 @@ else if (this.hasHmmPartnerFilterEnabled(accountId)) { else { trackerMotions.addAll(filteredOriginalMotions); } + } else { trackerMotions.addAll(filteredOriginalMotions); From 4f7bbd2782b2dfb303cc4aa41e0544dcd0009e85 Mon Sep 17 00:00:00 2001 From: Jarred Heinrich Date: Wed, 1 Feb 2017 13:55:53 -0800 Subject: [PATCH 2/4] added index safeguards --- .../core/util/MotionMaskPartnerFilter.java | 28 +++++++++++-------- .../InstrumentedTimelineProcessor.java | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/suripu-core/src/main/java/com/hello/suripu/core/util/MotionMaskPartnerFilter.java b/suripu-core/src/main/java/com/hello/suripu/core/util/MotionMaskPartnerFilter.java index ea59a69ee..dfce9a8b2 100644 --- a/suripu-core/src/main/java/com/hello/suripu/core/util/MotionMaskPartnerFilter.java +++ b/suripu-core/src/main/java/com/hello/suripu/core/util/MotionMaskPartnerFilter.java @@ -16,20 +16,20 @@ public class MotionMaskPartnerFilter { final private static int UNCERTAINTY_SECS = 1; //add uncertainty - public static List partnerFiltering(final List originalTrackerMotions, - final List originalPartnerMotions){ + public static List partnerFiltering(final List trackerMotions, + final List partnerMotions){ - if (originalTrackerMotions.isEmpty() || originalPartnerMotions.isEmpty()){ - return originalTrackerMotions; + if (trackerMotions.isEmpty() || partnerMotions.isEmpty()){ + return trackerMotions; } //requires motion mask - if(!originalTrackerMotions.get(0).motionMask.isPresent() ||!originalPartnerMotions.get(0).motionMask.isPresent()){ - return originalTrackerMotions; + if(!trackerMotions.get(0).motionMask.isPresent() ||!partnerMotions.get(0).motionMask.isPresent()){ + return trackerMotions; } - final PartnerMotionTimeSeries originalMotionTimeSeries= PartnerMotionTimeSeries.create(originalTrackerMotions,originalPartnerMotions); + final PartnerMotionTimeSeries originalMotionTimeSeries= PartnerMotionTimeSeries.create(trackerMotions,partnerMotions); final List originalTrackerMotionTimeSeriesWithPartner = originalMotionTimeSeries.groupByLeft(); - final List trackerMotions = new ArrayList<>(); + final List filteredTrackerMotions = new ArrayList<>(); final List removedTrackerMotionsWithPartnerMotion = new ArrayList<>(); @@ -42,10 +42,14 @@ public static List partnerFiltering(final List ori boolean partnerMovedAtSecond = false; if (trackerMotionIterator.hasPrevious()){ final int interMotionTimeDiffSeconds =(int) (trackerMotionWithPartner.trackerMotion.timestamp - previousTrackerMotionWithPartnerMotion.trackerMotion.timestamp )/ DateTimeConstants.MILLIS_PER_SECOND; - if (interMotionTimeDiffSeconds <= 60 && interMotionTimeDiffSeconds > 0) { + if (interMotionTimeDiffSeconds <= 60 && interMotionTimeDiffSeconds > 1 && previousTrackerMotionWithPartnerMotion.partnerMotionAtSeconds.size() > interMotionTimeDiffSeconds - 1) { partnerMovedAtSecond= previousTrackerMotionWithPartnerMotion.partnerMotionAtSeconds.get(interMotionTimeDiffSeconds - 1).didPartnerMove; } } + if(trackerMotionWithPartner.partnerMotionAtSeconds.size()==0){ + filteredTrackerMotions.add(trackerMotionWithPartner.trackerMotion); + continue; + } PartnerMotionAtSecond motionAtNextSecond = trackerMotionWithPartner.partnerMotionAtSeconds.get(0); boolean partnerMovedAtNextSecond = motionAtNextSecond.didPartnerMove; @@ -58,7 +62,7 @@ public static List partnerFiltering(final List ori }else if(trackerMotionIterator.hasNext()){ TrackerMotionWithPartnerMotion nextTrackerMotionWithPartner = trackerMotionIterator.next(); final int interMotionTimeDiffSeconds =(int) (nextTrackerMotionWithPartner.trackerMotion.timestamp - trackerMotionWithPartner.trackerMotion.timestamp)/ DateTimeConstants.MILLIS_PER_SECOND; - if (interMotionTimeDiffSeconds <= 60 && interMotionTimeDiffSeconds > 0) { + if (interMotionTimeDiffSeconds <= 60 && interMotionTimeDiffSeconds > 0 && !nextTrackerMotionWithPartner.partnerMotionAtSeconds.isEmpty()) { motionAtNextSecond = nextTrackerMotionWithPartner.partnerMotionAtSeconds.get(DateTimeConstants.SECONDS_PER_MINUTE - interMotionTimeDiffSeconds); partnerMovedAtNextSecond = motionAtNextSecond.didPartnerMove; } @@ -80,13 +84,13 @@ public static List partnerFiltering(final List ori if (userMovement){ - trackerMotions.add(trackerMotionWithPartner.trackerMotion); + filteredTrackerMotions.add(trackerMotionWithPartner.trackerMotion); }else{ removedTrackerMotionsWithPartnerMotion.add(trackerMotionWithPartner); } previousTrackerMotionWithPartnerMotion = trackerMotionWithPartner; } - return trackerMotions; + return filteredTrackerMotions; } } diff --git a/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java b/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java index c1eabc9fa..9709cbfe3 100644 --- a/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java +++ b/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java @@ -535,7 +535,7 @@ protected Optional getSensorData(final long accountId,final D trackerMotions.addAll(filteredOriginalMotions); } } - else if (this.hasMotionMaskPartnerFilter(accountId) && originalTrackerMotions.get(0).motionMask.isPresent()){ + else if (this.hasMotionMaskPartnerFilter(accountId) && filteredOriginalMotions.get(0).motionMask.isPresent() && filteredOriginalPartnerMotions.get(0).motionMask.isPresent()){ trackerMotions.addAll(MotionMaskPartnerFilter.partnerFiltering(filteredOriginalMotions, filteredOriginalPartnerMotions)); LOGGER.info("action=using-motion-mask-partner-filter account_id={} removed-motions={}", accountId, filteredOriginalMotions.size() - trackerMotions.size()); } From c9ac4d3ee4f664539a81774756028d2ccb7a1b7b Mon Sep 17 00:00:00 2001 From: Jarred Heinrich Date: Wed, 15 Feb 2017 14:56:35 -0800 Subject: [PATCH 3/4] motion mask partner filter now in addition to other filters --- .../InstrumentedTimelineProcessor.java | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java b/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java index 9709cbfe3..9663c1582 100644 --- a/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java +++ b/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java @@ -11,6 +11,7 @@ import com.hello.suripu.core.algorithmintegration.AlgorithmFactory; import com.hello.suripu.core.algorithmintegration.NeuralNetEndpoint; import com.hello.suripu.core.algorithmintegration.OneDaysSensorData; +import com.hello.suripu.core.algorithmintegration.OneDaysTrackerMotion; import com.hello.suripu.core.algorithmintegration.TimelineAlgorithm; import com.hello.suripu.core.algorithmintegration.TimelineAlgorithmResult; import com.hello.suripu.core.db.AccountReadDAO; @@ -53,7 +54,6 @@ import com.hello.suripu.core.models.TimelineResult; import com.hello.suripu.core.models.TrackerMotion; import com.hello.suripu.core.models.UserBioInfo; -import com.hello.suripu.core.algorithmintegration.OneDaysTrackerMotion; import com.hello.suripu.core.models.timeline.v2.TimelineLog; import com.hello.suripu.core.processors.FeatureFlippedProcessor; import com.hello.suripu.core.processors.PartnerMotion; @@ -519,7 +519,7 @@ protected Optional getSensorData(final long accountId,final D return Optional.absent(); } - final List trackerMotions = Lists.newArrayList(); + List trackerMotions = Lists.newArrayList(); if (!filteredOriginalPartnerMotions.isEmpty()) { @@ -535,10 +535,6 @@ protected Optional getSensorData(final long accountId,final D trackerMotions.addAll(filteredOriginalMotions); } } - else if (this.hasMotionMaskPartnerFilter(accountId) && filteredOriginalMotions.get(0).motionMask.isPresent() && filteredOriginalPartnerMotions.get(0).motionMask.isPresent()){ - trackerMotions.addAll(MotionMaskPartnerFilter.partnerFiltering(filteredOriginalMotions, filteredOriginalPartnerMotions)); - LOGGER.info("action=using-motion-mask-partner-filter account_id={} removed-motions={}", accountId, filteredOriginalMotions.size() - trackerMotions.size()); - } else if (this.hasHmmPartnerFilterEnabled(accountId)) { LOGGER.info("using hmm partner filter"); try { @@ -558,10 +554,25 @@ else if (this.hasHmmPartnerFilterEnabled(accountId)) { trackerMotions.addAll(filteredOriginalMotions); } + //motion mask filtering additive with other filters + if (this.hasMotionMaskPartnerFilter(accountId) && filteredOriginalMotions.get(0).motionMask.isPresent() && filteredOriginalPartnerMotions.get(0).motionMask.isPresent()){ + int t1 = trackerMotions.size(); + System.out.print(trackerMotions.size()); + System.out.print(", "); + trackerMotions = MotionMaskPartnerFilter.partnerFiltering(trackerMotions, filteredOriginalPartnerMotions); + LOGGER.info("action=using-motion-mask-partner-filter account_id={} removed-motions={}", accountId, filteredOriginalMotions.size() - trackerMotions.size()); + System.out.print(trackerMotions.size()); + System.out.print(","); + System.out.print(t1- trackerMotions.size()); + System.out.println(); + } + trackerMotions.clear(); + } else { trackerMotions.addAll(filteredOriginalMotions); } + trackerMotions.clear(); if (trackerMotions.isEmpty()) { LOGGER.debug("No tracker motion data ID for account_id = {} and day = {}", accountId, starteTimeLocalUTC); From fe87d0c8f1b4bcea7b351567515caba41fde333b Mon Sep 17 00:00:00 2001 From: Jarred Heinrich Date: Wed, 15 Feb 2017 15:03:44 -0800 Subject: [PATCH 4/4] removed print statements --- .../timeline/InstrumentedTimelineProcessor.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java b/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java index 9663c1582..f5513606b 100644 --- a/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java +++ b/suripu-coredropwizard/src/main/java/com/hello/suripu/coredropwizard/timeline/InstrumentedTimelineProcessor.java @@ -556,23 +556,14 @@ else if (this.hasHmmPartnerFilterEnabled(accountId)) { //motion mask filtering additive with other filters if (this.hasMotionMaskPartnerFilter(accountId) && filteredOriginalMotions.get(0).motionMask.isPresent() && filteredOriginalPartnerMotions.get(0).motionMask.isPresent()){ - int t1 = trackerMotions.size(); - System.out.print(trackerMotions.size()); - System.out.print(", "); trackerMotions = MotionMaskPartnerFilter.partnerFiltering(trackerMotions, filteredOriginalPartnerMotions); LOGGER.info("action=using-motion-mask-partner-filter account_id={} removed-motions={}", accountId, filteredOriginalMotions.size() - trackerMotions.size()); - System.out.print(trackerMotions.size()); - System.out.print(","); - System.out.print(t1- trackerMotions.size()); - System.out.println(); } - trackerMotions.clear(); } else { trackerMotions.addAll(filteredOriginalMotions); } - trackerMotions.clear(); if (trackerMotions.isEmpty()) { LOGGER.debug("No tracker motion data ID for account_id = {} and day = {}", accountId, starteTimeLocalUTC);