Skip to content

Anomaly detection #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions examples/Anomaly_Detection/Anomaly_Detection.ino
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/**
* Anomaly detection
* Detect when the frame changes by a reasonable amount
*
* BE SURE TO SET "TOOLS > CORE DEBUG LEVEL = DEBUG"
* to turn on debug messages
*/
#include <eloquent_esp32cam.h>
#include <eloquent_esp32cam/anomaly/detection.h>

using eloq::camera;
using eloq::anomaly::detection;

/**
*
*/
void setup() {
delay(3000);
Serial.begin(115200);
Serial.println("___ANOMALY DETECTION___");

// camera settings
// replace with your own model!
camera.pinout.xiao();
camera.brownout.disable();
camera.resolution.vga();
camera.quality.high();

// configure anomaly detection
detection.skip(4);
// the higher the stride, the faster the detection
// the higher the stride, the less the granularity
detection.stride(1);
// the higher the threshold, the less the sensitivity
// (at pixel level)
detection.threshold(5);
// the higher the detectionRatio, the less the sensitivity
// (at image level, from 0 to 1)
detection.detectionRatio(0.5);
// the higher the referenceRatio, the more the reference image can change over time
// (at image level, from 0 to 1)
detection.referenceRatio(0.2);
// optionally, you can enable rate limiting (aka debounce)
// anomaly won't trigger more often than the specified frequency
//detection.rate.atMostOnceEvery(5).seconds();

// init camera
while (!camera.begin().isOk())
Serial.println(camera.exception.toString());

Serial.println("Camera OK");
Serial.println("Awaiting anomaly...");
}

/**
*
*/
void loop() {

// Don't run more often than the time for an anomaly to come into view as the reference image can 'drift' away from 'normal'
delay(1000);
// capture picture
if (!camera.capture().isOk()) {
Serial.println(camera.exception.toString());
return;
}

// run anomaly detection
if (!detection.run().isOk()) {
Serial.println(detection.exception.toString());
return;
}

// on anomaly, perform action
if (detection.triggered()) {
Serial.print("Anomaly detected: "); Serial.println(detection.movingRatio);
}
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/**
* Run anomaly detection at low resolution.
* On anomaly, capture frame at higher resolution
* for SD storage.
*
* BE SURE TO SET "TOOLS > CORE DEBUG LEVEL = INFO"
* to turn on debug messages
*/
#include <eloquent_esp32cam.h>
#include <eloquent_esp32cam/anomaly/detection.h>

using eloq::camera;
using eloq::anomaly::detection;


/**
*
*/
void setup() {
delay(3000);
Serial.begin(115200);
Serial.println("___ANOMALY DETECTION + SWITCH RESOLUTION___");

// camera settings
// replace with your own model!
camera.pinout.freenove_s3();
camera.brownout.disable();
camera.resolution.vga();
camera.quality.high();

// see example of anomaly detection for config values
detection.skip(5);
detection.stride(1);
detection.threshold(5);
// the higher the detectionRatio, the less the sensitivity
// (at image level, from 0 to 1)
detection.detectionRatio(0.5);
// the higher the referenceRatio, the more the reference image can change over time
// (at image level, from 0 to 1)
detection.referenceRatio(0.2);

// init camera
while (!camera.begin().isOk())
Serial.println(camera.exception.toString());

Serial.println("Camera OK");
Serial.println("Awaiting for anomaly...");
}

/**
*
*/
void loop() {
// Don't run more often than the time for an anomaly to come into view as the reference image can 'drift' away from 'normal'
delay(1000);
// capture picture
if (!camera.capture().isOk()) {
Serial.println(camera.exception.toString());
return;
}

// run anomaly detection
if (!detection.run().isOk()) {
Serial.println(detection.exception.toString());
return;
}

// on anomaly, perform action
if (detection.triggered()) {
Serial.printf(
"Anomaly of %.2f detected on frame of size %dx%d (%d bytes)\n",
detection.movingRatio,
camera.resolution.getWidth(),
camera.resolution.getHeight(),
camera.getSizeInBytes()
);

Serial.println("Taking photo of anomaly at higher resolution");

camera.resolution.at(FRAMESIZE_UXGA, []() {
Serial.printf(
"Switched to higher resolution: %dx%d. It took %d ms to switch\n",
camera.resolution.getWidth(),
camera.resolution.getHeight(),
camera.resolution.benchmark.millis()
);

camera.capture();

Serial.printf(
"Frame size is now %d bytes\n",
camera.getSizeInBytes()
);

// save to SD...
});

Serial.println("Resolution switched back to VGA");
}
}
91 changes: 91 additions & 0 deletions src/eloquent_esp32cam/anomaly/daemon.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
#ifndef ELOQUENT_ESP32CAM_ANOMALY_DAEMON_H
#define ELOQUENT_ESP32CAM_ANOMALY_DAEMON_H

#include <functional>
#include "../camera/camera.h"
#include "../extra/esp32/multiprocessing/thread.h"

using eloq::camera;
using Eloquent::Extra::Esp32::Multiprocessing::Thread;
using OnAnomalyCallback = std::function<void(void)>;


namespace Eloquent {
namespace Esp32cam {
namespace Anomaly {
/**
* Run anomaly detection in a task
*
* @class Daemon
* @author jksemple
* @date 11/07/2024
* @file daemon.h
* @brief
*/
template<typename T>
class Daemon {
public:
Thread thread;

/**
* Constructor
*
* @brief
*/
Daemon(T* detection) :
thread("AnomalyDetection"),
_detection(detection) {

}

/**
* Run function when a difference from 'normal' is detected
*
* @brief
* @param callback
*/
void onAnomaly(OnAnomalyCallback callback) {
_onAnomaly = callback;
}

/**
* Start anomaly detection in background
*
* @brief
*/
void start() {
thread
.withArgs((void*) this)
.withStackSize(5000)
.run([](void *args) {
Daemon *self = (Daemon*) args;

delay(3000);

while (true) {
yield();
delay(1);

if (!camera.capture().isOk())
continue;

if (!self->_detection->run().isOk())
continue;

if (!self->_detection->triggered())
continue;

self->_onAnomaly();
}
});
}

protected:
T *_detection;
OnAnomalyCallback _onAnomaly;
};
}
}
}

#endif
214 changes: 214 additions & 0 deletions src/eloquent_esp32cam/anomaly/detection.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
#ifndef ELOQUENT_ESP32CAM_ANOMALY_DETECTION
#define ELOQUENT_ESP32CAM_ANOMALY_DETECTION

#include <dl_image.hpp>
#include "../extra/exception.h"
#include "../extra/time/benchmark.h"
#include "../extra/time/rate_limit.h"
#include "../extra/pubsub.h"
#include "./daemon.h"

using eloq::camera;
using Eloquent::Error::Exception;
using Eloquent::Extra::Time::Benchmark;
using Eloquent::Extra::Time::RateLimit;
#if defined(ELOQUENT_EXTRA_PUBSUB_H)
using Eloquent::Extra::PubSub;
#endif


namespace Eloquent {
namespace Esp32cam {
namespace Anomaly {
/**
* Detect anomaly using "fast" algorithm
*/
class Detection {
public:
float movingRatio;
Exception exception;
Benchmark benchmark;
RateLimit rate;
Daemon<Detection> daemon;
#if defined(ELOQUENT_EXTRA_PUBSUB_H)
PubSub<Detection> mqtt;
#endif

/**
*
*/
Detection() :
_stride(4),
_threshold(5),
_lowerDetectionRatio(0.2),
_upperDetectionRatio(0.5),
_referenceRatio(0.05),
_reference(NULL),
_skip(5),
movingRatio(0),
daemon(this),
#if defined(ELOQUENT_EXTRA_PUBSUB_H)
mqtt(this),
#endif
exception("AnomalyDetection") {

}

/**
* Set detection stride.
* The greater the value, the less accurate.
* The greater the value, the faster.
*/
void stride(uint8_t stride) {
_stride = stride;
}

/**
* Set detection sensitivity (pixel level).
* The greater the value, the less sensitive the detection.
*/
void threshold(uint8_t threshold) {
_threshold = threshold;
}

/**
* @brief Skip first frames (to avoid false detection)
* @param skip
*/
void skip(uint8_t skip) {
_skip = skip;
}

/**
* Set reference image sensitivity (image level).
* The greater the value, the more the reference image can vary over time.
*/
void referenceRatio(float ratio) {
_referenceRatio = ratio;
}

/**
* Set detection sensitivity (image level).
* The greater the value, the less sensitive the detection.
*/
void lowerDetectionRatio(float ratio) {
_lowerDetectionRatio = ratio;
}

/**
* Set maximum detection sensitivity (image level).
* This protects against false detections when the light level across the whole image changes
* Useful when you know objects being detected will never fill more than a fraction of the image
*/
void upperDetectionRatio(float ratio) {
_upperDetectionRatio = ratio;
}
/**
* Test if anomaly triggered
*/
inline bool triggered() {
return movingRatio >= _lowerDetectionRatio && movingRatio <= _upperDetectionRatio;
}

Exception& setReference() {
// convert JPEG to RGB565
if (!camera.rgb565.convert().isOk())
return camera.rgb565.exception;

if (_reference == NULL) {
_reference = (uint16_t*) ps_malloc(camera.rgb565.length * sizeof(uint16_t));
}
copy(camera.rgb565);

return exception.clear();
}
/**
*
*/
Exception& run() {
// skip fre first frames
if (_skip > 0 && _skip-- > 0)
return exception.set(String("Still ") + _skip + " frames to skip...");

// convert JPEG to RGB565
// this reduces the frame to 1/8th
if (!camera.rgb565.convert().isOk())
return camera.rgb565.exception;

// first frame, only copy frame to prev
if (_reference == NULL) {
_reference = (uint16_t*) malloc(camera.rgb565.length * sizeof(uint16_t));
copy(camera.rgb565);

return exception.set("First frame, can't detect anomaly").soft();
}

benchmark.timeit([this]() {
int movingPoints = dl::image::get_moving_point_number(
camera.rgb565.data,
_reference,
camera.rgb565.height,
camera.rgb565.width,
_stride,
_threshold
);

movingRatio = ((float) movingPoints) / camera.rgb565.length * _stride * _stride;
if (movingRatio < _referenceRatio) {
ESP_LOGD("AnomalyDetection", "Replacing reference frame - referenceRatio = %.2f", movingRatio);
copy(camera.rgb565);
}
});
ESP_LOGD("AnomalyDetection", "moving points ratio: %.2f", movingRatio);

// rate limit
if (triggered() && !rate)
return exception.set(rate.getRetryInMessage()).soft();

if (triggered())
rate.touch();

return exception.clear();
}
/**
* @brief Convert to JSON
*/
String toJSON() {
return String("{\"anomaly\":") + (triggered() ? "true" : "false") + "}";
}

/**
* @brief Test if an MQTT message should be published
*/
bool shouldPub() {
return triggered();
}

protected:
uint8_t _skip;
uint16_t *_reference;
uint8_t _stride;
uint8_t _threshold;
float _lowerDetectionRatio;
float _upperDetectionRatio;
float _referenceRatio;

/**
*
*/
template<typename Frame>
void copy(Frame frame) {
memcpy((uint8_t*) _reference, (uint8_t*) frame.data, frame.length * sizeof(uint16_t));
}
};
}
}
}

namespace eloq {
namespace anomaly {
static Eloquent::Esp32cam::Anomaly::Detection detection;
}
}

#endif
158 changes: 158 additions & 0 deletions src/eloquent_esp32cam/anomaly/roi_detection.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
#ifndef ELOQUENT_ESP32CAM_ANOMALY_ROI_DETECTION
#define ELOQUENT_ESP32CAM_ANOMALY_ROI_DETECTION

#include "./detection.h"

namespace Eloquent {
namespace Esp32cam {
namespace Anomaly {
/**
* Perform anomaly detection on Region of Interest
*/
class RoI : public Detection {
public:
struct {
uint16_t x;
uint16_t y;
uint16_t width;
uint16_t height;
uint16_t x1;
uint16_t x2;
uint16_t y1;
uint16_t y2;
} coords;

/**
*
*/
RoI() :
_x(0),
_y(0),
_w(0),
_h(0) {

}

/**
* Set x coordinate (top-left corner)
*/
void x(float x) {
_x = x;
}

/**
* Set y coordinate (top-left corner)
*/
void y(float y) {
_y = y;
}

/**
* Set width of RoI
*/
void width(float width) {
_w = width;
}

/**
* Set height of RoI
*/
void height(float height) {
_h = height;
}

/**
*
*/
void updateCoords(uint16_t width, uint16_t height) {
coords.x = max<int>(0, _x < 1 ? _x * width : _x);
coords.y = max<int>(0, _y < 1 ? _y * height : _y);
coords.width = min<int>(width - coords.x, _w < 1 ? _w * width : _w);
coords.height = min<int>(height - coords.y, _h < 1 ? _h * height : _h);
coords.x1 = coords.x;
coords.y1 = coords.y;
coords.x2 = coords.x + coords.width;
coords.y2 = coords.y + coords.height;
}

/**
* Detect anomaly
*/
template<typename Frame>
Exception& update(Frame& frame) {
if (!_w || !_h)
return exception.set("You MUST set a width and height for the RoI");

if (_reference == NULL) {
_reference = (uint8_t*) ps_malloc(_w * _h * sizeof(uint16_t));
_roi = (uint8_t*) ps_malloc(_w * _h * sizeof(uint16_t));
copy(frame, _reference);

return exception.set("First frame, can't detect anomaly").soft();
}

updateCoords(frame.width, frame.height);

benchmark.timeit([this, &frame]() {
copy(frame, _roi);

int movingPoints = dl::image::get_moving_point_number((uint16_t *) _roi, (uint16_t*) _reference, coords.height, coords.width, _stride, _threshold);
movingRatio = ((float) movingPoints) / sizeof(_roi) * _stride * _stride;
memcpy(_reference, _roi, sizeof(_reference));
});
if (movingRatio < _referenceRatio) {
// update reference
copy(frame, _reference);
}

ESP_LOGD(
"RoI AnomalyDetection",
"roi: (x=%d, y=%d, width=%d, height=%d). moving points: %.2f%%",
coords.x,
coords.y,
coords.width,
coords.height,
moving_ratio
);
if (triggered() && !rate_limiter)
return exception.set(rate.getRetryInMessage()).soft();

if (triggered())
rate_limiter.touch();

return exception.clear();
}


protected:
float _x;
float _y;
float _w;
float _h;
uint8_t *_roi;

/**
* Copy RoI of frame into buffer
*/
template<typename Frame>
void copy(Frame& frame, uint8_t *dest) {
for (int i = coords.y1; i < coords.y2; i++)
memcpy(
dest + coords.width * (i - coords.y1) * sizeof(uint16_t),
frame.data + (frame.width * i + coords.x) * sizeof(uint16_t),
coords.width * sizeof(uint16_t)
);
}
};
}
}
}

namespace eloq {
namespace anomaly {
// create class alias
class RoI : public Eloquent::Esp32cam::Anomaly::RoI {};
}
}

#endif
10 changes: 6 additions & 4 deletions src/eloquent_esp32cam/camera/rgb_565.h
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@ namespace Eloquent {
size_t length;
size_t width;
size_t height;
jpg_scale_t scaling;

/**
*
@@ -30,7 +31,8 @@ namespace Eloquent {
camera(cam),
length(0),
width(0),
height(0) {
height(0),
scaling(JPG_SCALE_8X) {
}

/**
@@ -94,8 +96,8 @@ namespace Eloquent {
return exception.set("Can't convert empty frame to RGB565");

if (!width) {
width = camera->resolution.getWidth() / 8;
height = camera->resolution.getHeight() / 8;
width = camera->resolution.getWidth() >> scaling;
height = camera->resolution.getHeight() >> scaling;
length = width * height;

ESP_LOGI("Camera", "Allocating %d bytes for %dx%d RGB565 image", length * 2, width, height);
@@ -106,7 +108,7 @@ namespace Eloquent {
return exception.set("Cannot allocate memory");

camera->mutex.threadsafe([this]() {
if (!jpg2rgb565(camera->frame->buf, camera->frame->len, (uint8_t*) data, JPG_SCALE_8X))
if (!jpg2rgb565(camera->frame->buf, camera->frame->len, (uint8_t*) data, scaling))
exception.set("Error converting frame from JPEG to RGB565");
});

2 changes: 0 additions & 2 deletions src/eloquent_esp32cam/motion/detection.h
Original file line number Diff line number Diff line change
@@ -131,7 +131,6 @@ namespace Eloquent {
movingRatio = ((float) movingPoints) / camera.rgb565.length * _stride * _stride;
copy(camera.rgb565);
});

ESP_LOGD("MotionDetection", "moving points ratio: %.2f", movingRatio);

// rate limit
@@ -143,7 +142,6 @@ namespace Eloquent {

return exception.clear();
}

/**
* @brief Convert to JSON
*/
52 changes: 36 additions & 16 deletions src/eloquent_esp32cam/transform/crop.h
Original file line number Diff line number Diff line change
@@ -42,8 +42,8 @@ namespace Eloquent {
_src.height = height;
_src.x1 = 0;
_src.y1 = 0;
_src.x2 = width;
_src.y2 = height;
_src.x2 = width - 1;
_src.y2 = height - 1;

return *this;
}
@@ -64,8 +64,8 @@ namespace Eloquent {
_out.height = height;
_out.x1 = 0;
_out.y1 = 0;
_out.x2 = width;
_out.y2 = height;
_out.x2 = width - 1;
_out.y2 = height - 1;

return *this;
}
@@ -93,13 +93,13 @@ namespace Eloquent {
*/
Crop& squash() {
_src.x1 = 0;
_src.x2 = _src.width;
_src.x2 = _src.width - 1;
_src.y1 = 0;
_src.y2 = _src.height;
_src.y2 = _src.height - 1;
_out.x1 = 0;
_out.x2 = _out.width;
_out.x2 = _out.width - 1;
_out.y1 = 0;
_out.y2 = _out.height;
_out.y2 = _out.height - 1;

return *this;
}
@@ -114,30 +114,50 @@ namespace Eloquent {

_src.x1 = dx;
_src.y1 = dy;
_src.x2 = _src.width - dx;
_src.y2 = _src.height - dy;
_src.x2 = _src.width - dx - 1;
_src.y2 = _src.height - dy - 1;
_out.x1 = 0;
_out.y1 = 0;
_out.x2 = _out.width;
_out.y2 = _out.height;
_out.x2 = _out.width - 1;
_out.y2 = _out.height - 1;

return *this;
}
else if (_out.width > _src.width) {
uint16_t dx = (_out.width - _src.width) / 2;
uint16_t dy = (_out.height - _src.height) / 2;

_out.x1 = dx;
_out.y1 = dy;
_out.x2 = _out.width - dx;
_out.y2 = _out.height - dy;
_out.x2 = _out.width - dx - 1;
_out.y2 = _out.height - dy - 1;
_src.x1 = 0;
_src.y1 = 0;
_src.x2 = _src.width;
_src.y2 = _src.height;
_src.x2 = _src.width - 1;
_src.y2 = _src.height - 1;
}

return *this;
}

/**
* Manually set crop area origin
* @param x
* @param y
* @return
*/
Crop& offset(int16_t x, int16_t y) {
if (x < 0) x += _src.width;
if (y < 0) y += _src.height;

_src.x1 = x;
_src.x2 = x + _out.width - 1;
_src.y1 = y;
_src.y2 = y + _out.height - 1;

return *this;
}

/**
* No interpolation
*/