Commit b28bebb5 authored by Gaurav Arora (gaa721)'s avatar Gaurav Arora (gaa721)
Browse files

Obscurdness, MousePosition added

parent d68509c4
......@@ -54,13 +54,15 @@ namespace FaceDetectionApp
#region mouse/keyboard events for hook
public void MouseMoved(object sender, MouseEventArgs e)
{
mouseLabel.Text = String.Format("Last mouse event: " + System.DateTime.Now.ToString());
mouseLabel.Text = String.Format("Last mouse event: " + System.DateTime.Now.ToString()) + " " + e.X + " " + e.Y;
if (e.Clicks > 0)
{
String.Format("Last mouse event: " + System.DateTime.Now.ToString());
}
FaceDetector.Instance.lastMouseEvent = System.DateTime.Now;
FaceDetector.Instance.mousePosition.X = e.X;
FaceDetector.Instance.mousePosition.Y = e.Y;
}
public void MyKeyDown(object sender, KeyEventArgs e)
......@@ -93,60 +95,18 @@ namespace FaceDetectionApp
}
public Rectangle getRectangle(WindowTitle windowHeader)
{
return new Rectangle(windowHeader.coordinates.Left, windowHeader.coordinates.Top, Math.Abs(windowHeader.coordinates.Right - windowHeader.coordinates.Left), Math.Abs(windowHeader.coordinates.Bottom - windowHeader.coordinates.Top));
}
public bool instesects(Rect RectA, Rect RectB)
{
return ((RectA.Left <= RectB.Right && RectA.Right >= RectB.Left &&
RectA.Top <= RectB.Bottom && RectA.Bottom >= RectB.Top));
}
public bool insideH(Rect RectA, Rect RectB)
{
return RectB.Right < RectA.Right && RectB.Left > RectA.Left && RectB.Top < RectA.Top && RectB.Bottom > RectB.Bottom;
}
public bool inside(Rect RectA, Rect RectB)
{
return insideH(RectA, RectB) || insideH(RectB,RectA);
}
private void F_ActiveWindowChanged(object sender, WindowTitle windowHeader, List<WindowTitle> windows, IntPtr hwnd)
{
bool Obscured = false;
String obsText = "";
Rectangle r1 = getRectangle(windowHeader);
foreach (WindowTitle w in windows)
{
if (!getRectangle(w).Contains(r1))
{
Console.WriteLine(w.ToString());
}
if (instesects(w.coordinates, windowHeader.coordinates) && (r1 != getRectangle(w)) && !getRectangle(w).Contains(r1) )
{
Obscured = true;
break;
}
}
actWinVis.Text = "Obscured: " + Obscured;
string text = string.Join("\n", windows);
actWinVis.Text = "Obscured: " + windowHeader.obscured ;
string text = string.Join(System.Environment.NewLine, windows);
textBox1.Text = "CURRENT WINDOW:" + windowHeader + "\r\n\r\nVISIBLE WINDOWS:\r\n" + text ;
if (!windowHeader.Title.Equals("Form1"))
{
......
......@@ -44,9 +44,12 @@ namespace FaceDetectionApp
{
JSONMessage.CurrectFaceState = FaceState.Face;
JSONMessage.CurrentTitle = FaceDetector.Instance.GetActiveWindowTitle();
FaceDetector.Instance.calculateObscurdness(ref JSONMessage.CurrentTitle,ref FaceDetector.Instance.Windows);
JSONMessage.VisibleWindows = FaceDetector.Instance.Windows;
JSONMessage.lastKeyboardEvent = FaceDetector.Instance.lastKeyboardEvent;
JSONMessage.lastMouseEvent = FaceDetector.Instance.lastMouseEvent;
JSONMessage.mousePosition = FaceDetector.Instance.mousePosition;
//write out to each connected listener
//Console.WriteLine(JSONMessage.ToString());
foreach (var s in sw) s.WriteLine(JSONMessage.ToString());
......@@ -61,9 +64,13 @@ namespace FaceDetectionApp
{
JSONMessage.CurrectFaceState = FaceState.NoFace;
JSONMessage.CurrentTitle = FaceDetector.Instance.GetActiveWindowTitle();
FaceDetector.Instance.calculateObscurdness(ref JSONMessage.CurrentTitle, ref FaceDetector.Instance.Windows);
JSONMessage.VisibleWindows = FaceDetector.Instance.Windows;
JSONMessage.lastKeyboardEvent = FaceDetector.Instance.lastKeyboardEvent;
JSONMessage.lastMouseEvent = FaceDetector.Instance.lastMouseEvent;
JSONMessage.mousePosition = FaceDetector.Instance.mousePosition;
//write out to each connected listener
foreach (var s in sw) s.WriteLine(JSONMessage.ToString());
} catch (Exception e) {
......@@ -77,8 +84,11 @@ namespace FaceDetectionApp
{
//write out to each connected listener
JSONMessage.CurrentTitle = FaceDetector.Instance.GetActiveWindowTitle();
FaceDetector.Instance.calculateObscurdness(ref JSONMessage.CurrentTitle, ref FaceDetector.Instance.Windows);
JSONMessage.VisibleWindows = FaceDetector.Instance.Windows;
JSONMessage.mousePosition = FaceDetector.Instance.mousePosition;
}
catch (Exception e)
{
......@@ -93,10 +103,14 @@ namespace FaceDetectionApp
try
{
JSONMessage.CurrentTitle = FaceDetector.Instance.GetActiveWindowTitle();
JSONMessage.VisibleWindows = FaceDetector.Instance.Windows;
FaceDetector.Instance.calculateObscurdness(ref JSONMessage.CurrentTitle, ref FaceDetector.Instance.Windows);
JSONMessage.VisibleWindows = FaceDetector.Instance.Windows;
JSONMessage.lastKeyboardEvent = FaceDetector.Instance.lastKeyboardEvent;
JSONMessage.lastMouseEvent = FaceDetector.Instance.lastMouseEvent;
foreach (var s in sw) s.WriteLine(JSONMessage.ToString());
JSONMessage.mousePosition = FaceDetector.Instance.mousePosition;
foreach (var s in sw) s.WriteLine(JSONMessage.ToString());
}
catch (Exception e)
{
......@@ -112,9 +126,12 @@ namespace FaceDetectionApp
try
{
JSONMessage.CurrentTitle = FaceDetector.Instance.GetActiveWindowTitle();
FaceDetector.Instance.calculateObscurdness(ref JSONMessage.CurrentTitle, ref FaceDetector.Instance.Windows);
JSONMessage.VisibleWindows = FaceDetector.Instance.Windows;
JSONMessage.lastKeyboardEvent = FaceDetector.Instance.lastKeyboardEvent;
JSONMessage.lastMouseEvent = FaceDetector.Instance.lastMouseEvent;
JSONMessage.mousePosition = FaceDetector.Instance.mousePosition;
foreach (var s in sw) s.WriteLine(JSONMessage.ToString());
}
catch (Exception e)
......@@ -129,9 +146,12 @@ namespace FaceDetectionApp
try
{
JSONMessage.CurrentTitle = FaceDetector.Instance.GetActiveWindowTitle();
FaceDetector.Instance.calculateObscurdness(ref JSONMessage.CurrentTitle, ref FaceDetector.Instance.Windows);
JSONMessage.VisibleWindows = FaceDetector.Instance.Windows;
JSONMessage.lastKeyboardEvent = FaceDetector.Instance.lastKeyboardEvent;
JSONMessage.lastMouseEvent = FaceDetector.Instance.lastMouseEvent;
JSONMessage.mousePosition = FaceDetector.Instance.mousePosition;
foreach (var s in sw) s.WriteLine(JSONMessage.ToString());
}
catch (Exception e)
......@@ -153,10 +173,6 @@ namespace FaceDetectionApp
f.FaceDetectedEvent += FaceDetectedCallback;
f.FaceLostEvent += FaceLostCallback;
f.ActiveWindowChanged += ActiveWindowChangedCallback;
JSONMessage = new JSONMessage(f.GetActiveWindowTitle(), f.Windows);
#pragma warning disable CS0618 // Type or member is obsolete
......
......@@ -23,12 +23,17 @@ namespace FaceDetectionClientConsole
public void changeInJSON()
{
label1.Text = FaceDetectionClientConsoel.Program.message.CurrectFaceState.ToString();
richTextBox1.Text = FaceDetectionClientConsoel.Program.message.CurrentTitle.ToString() + FaceDetectionClientConsoel.Program.message.VisibleWindows.ToString();
label2.Text = FaceDetectionClientConsoel.Program.message.lastMouseEvent.ToString();
label3.Text = FaceDetectionClientConsoel.Program.message.lastKeyboardEvent.ToString();
String text = String.Join("\n", FaceDetectionClientConsoel.Program.message.VisibleWindows);
richTextBox1.Text = FaceDetectionClientConsoel.Program.message.mousePosition.X + " " + FaceDetectionClientConsoel.Program.message.mousePosition.Y + FaceDetectionClientConsoel.Program.message.CurrentTitle.ToString() + text ;
label2.Text = FaceDetectionClientConsoel.Program.message.lastMouseEvent.ToString();
label3.Text = FaceDetectionClientConsoel.Program.message.lastKeyboardEvent.ToString();
}
......
......@@ -29,9 +29,9 @@ namespace FaceDetectionClientConsoel
{
Init();
Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false);
Application.Run(new FaceDetectionClientConsole.Form2());
Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false);
Application.Run(new FaceDetectionClientConsole.Form1());
}
......@@ -63,7 +63,7 @@ namespace FaceDetectionClientConsoel
String msg = sr.ReadLine();
message = new JSONMessage(msg);
Console.WriteLine(msg);
// Console.WriteLine(msg);
changes();
......
......@@ -24,7 +24,12 @@ namespace FaceDetectionMain
public delegate void FaceLostHandler();
public delegate void AttentionCheck();
public delegate void ActiveWindowChangedHandler(object sender, WindowTitle windowHeader, List<WindowTitle> Windows, IntPtr hwnd);
public struct mousePos
{
public int X;
public int Y;
};
public class FaceDetector
{
#region props
......@@ -34,22 +39,22 @@ namespace FaceDetectionMain
public FrameCapturedHandler FrameCapturedEvent;
public FaceDetectedHandler FaceDetectedEvent;
public FaceLostHandler FaceLostEvent;
public AttentionCheck AttentionChecker;
public AttentionCheck AttentionChecker;
private Capture grabber;
private Mat currentFrame;
private const string Classifier = "/haarcascade_frontalface_default.xml";
//private CascadeClassifier faceCascadeClassifier;
private Rectangle[] facesDetected;
private bool hasFace = false;
public bool HighlightFace = false;
public Bitmap Image { get { if (currentFrame != null) return currentFrame.Bitmap; return null; } }
public bool HasFace { get { return hasFace; } }
public bool IsCapturing { get { return (grabber != null && grabber.Grab()); } }
#endregion
......@@ -65,7 +70,7 @@ namespace FaceDetectionMain
[DllImport("user32.dll")]
static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
[DllImport("user32.dll")]
static extern bool GetWindowRect(IntPtr hwnd, out RECT lpRect);
......@@ -94,7 +99,7 @@ namespace FaceDetectionMain
public static extern IntPtr FindWindow(string strClassName, string strWindowName);
[DllImport("user32.dll")]
public static extern bool GetWindowRect(IntPtr hwnd, ref Rect rectangle);
......@@ -161,6 +166,9 @@ namespace FaceDetectionMain
if (grabber != null) grabber.Dispose();
//faceCascadeClassifier.Dispose();
UnhookWinEvent(m_hhook);
UnhookWinEvent(m_hhook1);
UnhookWinEvent(m_hhook2);
}
#endregion
......@@ -177,7 +185,7 @@ namespace FaceDetectionMain
CvInvoke.CvtColor(currentFrame, gray, Emgu.CV.CvEnum.ColorConversion.Bgr2Gray);
CvInvoke.EqualizeHist(gray, gray);
using(CascadeClassifier fc = new CascadeClassifier(Directory.GetCurrentDirectory() + Classifier))
using (CascadeClassifier fc = new CascadeClassifier(Directory.GetCurrentDirectory() + Classifier))
facesDetected = fc.DetectMultiScale(gray, 1.1, 10, new Size(20, 20));
}
......@@ -187,7 +195,7 @@ namespace FaceDetectionMain
if (FaceDetectedEvent != null) { hasFace = true; FaceDetectedEvent(); }
if (FrameCapturedEvent != null)
if(!HighlightFace) FrameCapturedEvent(Image, facesDetected);
if (!HighlightFace) FrameCapturedEvent(Image, facesDetected);
else FrameCapturedEvent(DrawFaces(currentFrame, facesDetected), facesDetected);
}
......@@ -196,23 +204,99 @@ namespace FaceDetectionMain
{
Image<Bgr, byte> ret = m.ToImage<Bgr, byte>();
foreach (Rectangle r in faces)
foreach (Rectangle r in faces)
ret.Draw(r, new Bgr(Color.White));
return ret.ToBitmap();
}
#endregion
#region Window
public Rectangle getRectangle(WindowTitle windowHeader)
{
return new Rectangle(windowHeader.coordinates.Left, windowHeader.coordinates.Top, Math.Abs(windowHeader.coordinates.Right - windowHeader.coordinates.Left), Math.Abs(windowHeader.coordinates.Bottom - windowHeader.coordinates.Top));
}
public bool instesects(Rect RectA, Rect RectB)
{
return ((RectA.Left <= RectB.Right && RectA.Right >= RectB.Left &&
RectA.Top <= RectB.Bottom && RectA.Bottom >= RectB.Top));
}
public void calculateObscurdness(ref WindowTitle windowHeader, ref List<WindowTitle> windows)
{
Rectangle r1 = getRectangle(windowHeader);
windowHeader.obscured = false;
foreach (WindowTitle w in windows)
{
if (w.Title.ToString() == "Program Manager")
{
if (instesects(w.coordinates, windowHeader.coordinates) && (r1 != getRectangle(w)) && !getRectangle(w).Contains(r1))
{
windowHeader.obscured = true;
break;
}
break;
}
}
for (int i = 0; i < windows.Count; i++)
{
WindowTitle w = windows[i];
w.obscured = false;
foreach (WindowTitle w1 in windows)
{
if (w.Title.ToString() != w1.Title.ToString())
{
if (instesects(w1.coordinates, w.coordinates) && (getRectangle(w1) != getRectangle(w)) && !getRectangle(w1).Contains(getRectangle(w)))
{
w.obscured = true;
break;
}
}
}
windows[i] = w;
}
}
void windowChanged(IntPtr hWinEventHook, uint eventType, IntPtr hwnd, int idObject, int idChild, uint dwEventThread, uint dwmsEventTime)
{
if (eventType == EVENT_SYSTEM_FOREGROUND) {
Windows = new List<WindowTitle>(); ;
if (eventType == EVENT_SYSTEM_FOREGROUND)
{
Windows = new List<WindowTitle>();
EnumWindows(callBackPtr, 0);
if (ActiveWindowChanged != null)
ActiveWindowChanged(this, GetActiveWindowTitle(hwnd), Windows, hwnd);
{
WindowTitle temp = GetActiveWindowTitle(hwnd);
calculateObscurdness(ref temp, ref Windows);
ActiveWindowChanged(this, temp, Windows, hwnd);
}
}
}
......@@ -223,16 +307,21 @@ namespace FaceDetectionMain
Windows = new List<WindowTitle>();
EnumWindows(callBackPtr, 0);
if (ActiveWindowChanged != null)
ActiveWindowChanged(this, GetActiveWindowTitle(hwnd), Windows, hwnd);
{
WindowTitle temp = GetActiveWindowTitle(hwnd);
calculateObscurdness(ref temp, ref Windows);
ActiveWindowChanged(this, temp, Windows, hwnd);
}
}
}
void windowMinimized(IntPtr hWinEventHook, uint eventType, IntPtr hwnd, int idObject, int idChild, uint dwEventThread, uint dwmsEventTime)
{
Console.WriteLine(eventType);
if (eventType == EVENT_SYSTEM_MINIMIZEEND)
{
......@@ -240,32 +329,39 @@ namespace FaceDetectionMain
Windows = new List<WindowTitle>();
EnumWindows(callBackPtr, 0);
if (ActiveWindowChanged != null)
ActiveWindowChanged(this, GetActiveWindowTitle(hwnd), Windows, hwnd);
{
WindowTitle temp = GetActiveWindowTitle(hwnd);
calculateObscurdness(ref temp, ref Windows);
ActiveWindowChanged(this, temp, Windows, hwnd);
}
}
}
private WindowTitle GetActiveWindowTitle(IntPtr hwnd) {
private WindowTitle GetActiveWindowTitle(IntPtr hwnd)
{
StringBuilder Buff = new StringBuilder(500);
GetWindowText(hwnd, Buff, Buff.Capacity);
StringBuilder Buff2 = new StringBuilder(500);
GetWindowModuleFileName(GetForegroundWindow(), Buff2, Buff2.Capacity);
Rect tempCoordinates = new Rect();
GetWindowRect(hwnd, ref tempCoordinates);
return new WindowTitle(Buff.ToString(), Buff2.ToString(), true,tempCoordinates);
return new WindowTitle(Buff.ToString(), Buff2.ToString(), true, tempCoordinates, false);
}
public WindowTitle GetActiveWindowTitle() {
public WindowTitle GetActiveWindowTitle()
{
StringBuilder Buff = new StringBuilder(500);
GetWindowText(GetForegroundWindow(), Buff, Buff.Capacity);
StringBuilder Buff2 = new StringBuilder(500);
GetWindowModuleFileName(GetForegroundWindow(), Buff2, Buff2.Capacity);
Rect tempCoordinates = new Rect();
GetWindowRect(GetForegroundWindow(), ref tempCoordinates);
return new WindowTitle(Buff.ToString(), Buff2.ToString(), true,tempCoordinates);
return new WindowTitle(Buff.ToString(), Buff2.ToString(), true, tempCoordinates, false);
}
public bool EnumWindowsCallback(IntPtr hwnd, int lParam) {
public bool EnumWindowsCallback(IntPtr hwnd, int lParam)
{
string title = GetActiveWindowTitle(hwnd).Title;
bool isVisible = IsWindowVisible(hwnd);
Rect tempCoordinates = new Rect();
......@@ -274,13 +370,13 @@ namespace FaceDetectionMain
GetWindowModuleFileName(GetForegroundWindow(), Buff2, Buff2.Capacity);
if (!string.IsNullOrEmpty(title) && string.Compare(title, "Default IME", true) != 0)
if (isVisible) //Comment if you want all states, not just visible
Windows.Add(new WindowTitle(title, Buff2.ToString(), isVisible, tempCoordinates));
Windows.Add(new WindowTitle(title, Buff2.ToString(), isVisible, tempCoordinates, false));
return true;
}
#endregion
#region attention
......@@ -295,27 +391,29 @@ namespace FaceDetectionMain
public DateTime lastMouseEvent = System.DateTime.Now;
public DateTime lastKeyboardEvent = System.DateTime.Now;
public mousePos mousePosition;
}
public enum FaceState { NoFace, Face }
[DataContract]
public class JSONMessage {
public class JSONMessage
{
[DataMember]
public WindowTitle CurrentTitle;
[DataMember]
public List<WindowTitle> VisibleWindows; //Change this to List<WindowTitle>
public List<WindowTitle> VisibleWindows;
[DataMember]
public FaceState CurrectFaceState;
[DataMember]
public DateTime lastMouseEvent;
[DataMember]
public DateTime lastKeyboardEvent;
[DataMember]
public mousePos mousePosition;
public JSONMessage(FaceState currentFaceState,WindowTitle currentTitle, List<WindowTitle> visibleWindows)
public JSONMessage(FaceState currentFaceState, WindowTitle currentTitle, List<WindowTitle> visibleWindows)
{
CurrentTitle = currentTitle;
VisibleWindows = visibleWindows;
......@@ -323,12 +421,14 @@ namespace FaceDetectionMain
}
public JSONMessage(WindowTitle currentTitle, List<WindowTitle> visibleWindows) {
public JSONMessage(WindowTitle currentTitle, List<WindowTitle> visibleWindows)
{
CurrentTitle = currentTitle;
VisibleWindows = visibleWindows;
}
public JSONMessage(string JSON) {
public JSONMessage(string JSON)
{
DataContractJsonSerializer ser = new DataContractJsonSerializer(typeof(JSONMessage));
MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(JSON));
JSONMessage obj = (JSONMessage)ser.ReadObject(ms);
......@@ -337,10 +437,12 @@ namespace FaceDetectionMain
this.CurrectFaceState = obj.CurrectFaceState;
this.lastMouseEvent = obj.lastMouseEvent;
this.lastKeyboardEvent = obj.lastKeyboardEvent;
this.mousePosition = obj.mousePosition;
}
public string ToJSON() {
public string ToJSON()
{
MemoryStream stream1 = new MemoryStream();
DataContractJsonSerializer ser = new DataContractJsonSerializer(typeof(JSONMessage));
ser.WriteObject(stream1, this);
......@@ -369,20 +471,22 @@ namespace FaceDetectionMain
public string FilePath;
public bool Visible;
public Rect coordinates;
public bool obscured;
public WindowTitle(string title, string fp, bool visible,Rect crd)
public WindowTitle(string title, string fp, bool visible, Rect crd, bool obs)
{
Title = title;
FilePath = fp;
Visible = visible;
coordinates = crd;
obscured = obs;
}