WSL + API support

Hawkeye.VisionBuilder.Workflow/DataTransfer/PythonModelProxyHTTP.cs

@@ -0,0 +1,6 @@
+namespace Hawkeye.VisionBuilder.Workflow.DataTransfer;
+
+public class PythonModelProxyHTTP
+{
+    
+}

PythonModelAPI/PythonModelAPI.py

@@ -0,0 +1,107 @@
+
+# app.py
+from fastapi import FastAPI, UploadFile, File, Body
+from fastapi.responses import JSONResponse
+import numpy as np
+from PIL import Image
+import io
+import tensorflow as tf
+
+app = FastAPI()
+
+class Service:
+    def __init__(self):
+        self.model = None
+        self.models={}
+        self.active_model = "default"
+
+    def activate_model(self, name):
+
+        self.model = self.models[name]
+        self.active_model = name
+        return 0
+
+    def load_model(self, path, name):
+        self.models[name] = tf.keras.models.load_model(path)
+        return 0
+
+    def get_accept_size(self):
+        print(self.model.layers[0].input)
+
+        return list(self.model.input_shape)
+
+    def get_output_size(self):
+        return list(self.model.layers[-1].output[0].shape[1:4])
+
+
+    def predict(self, data: bytes):
+        target_size = self.get_accept_size()
+        image = np.reshape(
+            np.frombuffer(data, dtype=np.uint8),
+            (target_size[1], target_size[2], target_size[3])
+        ) / 255.0
+
+        predictions = self.model.predict(np.array([image]))
+
+        if predictions[0].ndim == 2:
+            classed = predictions[0]
+        else:
+            classed = tf.argmax(predictions[0], axis=2)
+
+        return JSONResponse({"result": classed.numpy().reshape(-1).tolist()})
+
+    def predict_raw(self, data):
+        target_size = self.get_accept_size()
+        image = np.reshape(np.frombuffer(data, dtype=np.uint8), (target_size[1], target_size[2], target_size[3])) / 255.
+
+        predictions = self.model(np.array([image])).numpy()
+
+        if predictions[0].ndim==2:
+            classed = predictions[0][:,:,np.newaxis].clip(0,255)
+            return (classed).astype(np.uint8).reshape(-1).tolist()
+        else:
+            classed = predictions[0][:,:,:]
+
+        return (classed*255).astype(np.uint8).reshape(-1).tolist()
+
+svc = Service()
+
+def _prep_raw_bytes_for_predict(buf: bytes) -> bytes:
+    """Decode JPEG/PNG -> RGB, resize to model size, return raw uint8 bytes HxWxC."""
+    h, w, c = svc.get_accept_size()[1:4]
+    img = Image.open(io.BytesIO(buf)).convert("RGB")
+    if img.size != (w, h):
+        img = img.resize((w, h), Image.BILINEAR)
+    arr = np.asarray(img, dtype=np.uint8)  # HxWx3 uint8
+    return arr.tobytes()
+
+@app.post("/predict")
+async def predict(file: UploadFile = File(...)):
+    buf = await file.read()
+    raw = _prep_raw_bytes_for_predict(buf)
+    return svc.predict(raw)
+
+@app.post("/predict_raw")
+async def predict_raw(file: UploadFile = File(...)):
+    buf = await file.read()
+    raw = _prep_raw_bytes_for_predict(buf)
+    return svc.predict_raw(raw)
+
+@app.post("/load_model")
+async def load_model(path: str = Body(...), name: str = Body(...)):
+    try:
+        svc.load_model(path, name)
+        return JSONResponse({"status": "success", "message": f"Model {name} loaded successfully."})
+    except Exception as e:
+        return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
+
+@app.post("/activate_model")
+async def activate_model(name: str = Body(...)):
+    try:
+        svc.activate_model(name)
+        return JSONResponse({"status": "success", "message": f"Model {name} activated successfully."})
+    except Exception as e:
+        return JSONResponse({"status": "error", "message": str(e)}, status_code=500)
+
+
+

PythonModelAPI/PythonModelAPI.pyproj

@@ -0,0 +1,35 @@
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="4.0">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>b4949493-c2cd-4786-860d-daacd39a662d</ProjectGuid>
+    <ProjectHome>.</ProjectHome>
+    <StartupFile>PythonModelAPI.py</StartupFile>
+    <SearchPath>
+    </SearchPath>
+    <WorkingDirectory>.</WorkingDirectory>
+    <OutputPath>.</OutputPath>
+    <Name>PythonModelAPI</Name>
+    <RootNamespace>PythonModelAPI</RootNamespace>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
+    <DebugSymbols>true</DebugSymbols>
+    <EnableUnmanagedDebugging>false</EnableUnmanagedDebugging>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
+    <DebugSymbols>true</DebugSymbols>
+    <EnableUnmanagedDebugging>false</EnableUnmanagedDebugging>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="PythonModelAPI.py" />
+  </ItemGroup>
+  <Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets" />
+  <!-- Uncomment the CoreCompile target to enable the Build command in
+       Visual Studio and specify your pre- and post-build commands in
+       the BeforeBuild and AfterBuild targets below. -->
+  <!--<Target Name="CoreCompile" />-->
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+</Project>

PythonModelAPI/Segment128Half.py

@@ -0,0 +1,210 @@
+from albumentations.augmentations.utils import P
+from PythonDataTransferMQRPC import PythonDataTransferMQRPC
+import albumentations as A
+import EgLib as El
+import tensorflow as tf
+import numpy as np
+from sklearn.model_selection import train_test_split
+from PIL import Image
+from tensorflow.keras.utils import Sequence
+import time
+
+class DataGenerator(Sequence):
+    def __init__(self, x_set, y_set, batch_size):
+        self.x, self.y = x_set, y_set
+        self.batch_size = batch_size
+
+    def __len__(self):
+        return int(np.ceil(len(self.x) / float(self.batch_size)))
+
+    def __getitem__(self, idx):
+        batch_x = self.x[idx * self.batch_size:(idx + 1) * self.batch_size]
+        batch_y = self.y[idx * self.batch_size:(idx + 1) * self.batch_size]
+        return batch_x, batch_y
+
+transform = A.Compose([
+
+    A.HorizontalFlip(p=0.5),
+    A.VerticalFlip(p=0.5),
+    A.Rotate(p=0.8),
+
+])
+classes = 25
+
+models = {}
+
+
+
+class HandlerClass:
+
+    def __init__(self):
+        self.model = None
+        self.active_model = None
+        self.val_dataset = None
+
+        self.train_dataset = None
+
+        self.Y_test = None
+        self.Y_train = None
+        self.X_train = None
+        self.X_test = None
+        self.const={}
+
+    def activate_model(self, name):
+
+        self.model = models[name]
+        self.active_model = name
+
+        
+
+        return 0
+
+    def start_learning(self, path):
+        print("start learning")
+        target_size = self.get_accept_size()
+        images, labels = El.load_hawkeye(path, target_size=(target_size[1], target_size[2]))
+
+        images_augmented = []
+        labels_augmented = []
+
+        for _ in range(15):
+            for i, l in zip(images, labels):
+                transformed = transform(image=i, mask=l)
+                images_augmented.append(transformed["image"])
+                labels_augmented.append(transformed["mask"])
+        images, labels = np.array(images_augmented), np.array(labels_augmented)
+        labels = np.squeeze(labels[:, :, :, 0])
+        labels = tf.one_hot(labels, classes)
+
+        images_normalized = images / 255.
+        self.X_train, self.X_test, self.Y_train, self.Y_test = train_test_split(images_normalized, labels.numpy(),
+                                                                                test_size=0.33, random_state=42)
+
+        self.train_dataset = DataGenerator(self.X_train, self.Y_train, 16)
+        self.val_dataset = DataGenerator(self.X_test, self.Y_test, 16)
+
+        return 0
+
+    def create_model(self, name, additional_layers, classes):
+        input_layer = layers.Input(shape=(128,128,3))
+        x = layers.Conv2D(16, 7,strides=1, padding='same',activation="relu")(input_layer)
+        x = layers.MaxPooling2D()(x)
+        x = residual_block(x,16,5)
+        x = layers.MaxPooling2D()(x)
+
+        for _ in range(additional_layers):
+            x = residual_block(x,16,3)
+
+        x = residual_block(x,16,3)
+        x = layers.UpSampling2D()(x)
+        x = residual_block(x,16,3)
+        x = layers.UpSampling2D()(x)
+        x = layers.Conv2D(9, 3, padding='same',activation="softmax")(x)
+
+
+        model = Model(inputs=input_layer,outputs=x)
+        model.summary()
+
+        model.compile(loss="categorical_crossentropy",optimizer=tf.optimizers.Adam(learning_rate=0.01),metrics=["accuracy"])
+
+    def save_model(self, path):
+        self.model.save(path)
+        return 0
+
+    def load_model(self, path, name):
+        models[name]=tf.keras.models.load_model(path)
+
+        
+        self.const[name]=None
+            
+        return 0
+
+    def get_accept_size(self):
+        return list(self.model.input_shape)
+
+    def get_output_size(self):
+        return list(self.model.layers[-1].output.shape[1:4])
+
+    def predict(self, data):
+        target_size = self.get_accept_size()
+        image = np.reshape(np.frombuffer(data, dtype=np.uint8), (target_size[1], target_size[2], target_size[3])) / 255.
+        if self.const[self.active_model] is not None:
+            predictions = self.model.predict([np.array([image]),self.const])
+        else:   
+            predictions = self.model.predict(np.array([image]))
+        
+        
+
+        if predictions[0].ndim==2:
+            classed = predictions[0]
+        else:
+            classed = tf.argmax(predictions[0], axis=2)
+
+        print(classed.shape)
+
+        return classed.numpy().reshape(-1).tolist()
+
+    def predict_anomaly(self,data):
+        target_size = self.get_accept_size()
+        image = np.reshape(np.frombuffer(data, dtype=np.uint8), (target_size[1], target_size[2], target_size[3])) / 255.
+        inputs=[np.array([image]),np.array([self.const[self.active_model]])]
+        
+        predictions = self.model.predict(inputs)
+        print(predictions[0][0][0].shape)
+        print(predictions[1].shape)
+        
+        image=predictions[0][0][0]
+                
+
+        normalized_image=(image-image.min())/(image.max()-image.min())
+        defect_probability=predictions[1][0][1]*100
+        squized_image=(normalized_image*255).reshape(-1)
+        
+        squized_prop=np.array([defect_probability*100]).astype(int)
+        
+        res = np.concatenate((squized_prop,(normalized_image*255).astype(int).reshape(-1))).astype(np.uint8).tolist()
+        print(len(res))
+        return res
+
+    def predict_raw(self, data):
+        target_size = self.get_accept_size()
+        image = np.reshape(np.frombuffer(data, dtype=np.uint8), (target_size[1], target_size[2], target_size[3])) / 255.
+
+        # measure time
+        strart_time=time.time()
+
+        if self.const[self.active_model] is not None:
+            predictions = self.model.predict([np.array([image]),self.const])
+        else:   
+            print("test")
+            predictions = self.model(np.array([image])).numpy()
+
+        print(time.time()-strart_time)
+
+        if predictions[0].ndim==2:
+            classed = predictions[0][:,:,np.newaxis].clip(0,255)
+            return (classed).astype(np.uint8).reshape(-1).tolist()
+        else:
+            classed = predictions[0][:,:,:]
+
+        print(classed.max())
+        print(classed.min())
+        return (classed*255).astype(np.uint8).reshape(-1).tolist()
+
+
+
+    def continue_learning(self):
+        print("continue learning")
+        schedule = tf.keras.callbacks.LearningRateScheduler(
+            lambda epoch: 0.001, verbose=0)
+
+        model_history = self.model.fit(self.train_dataset, epochs=1,
+                                       validation_data=self.val_dataset,
+                                       callbacks=[schedule])
+        tf.keras.backend.clear_session()
+        return model_history.history["val_loss"][0]
+
+
+data_transfer = PythonDataTransferMQRPC()
+
+data_transfer.wait_for_data_and_process(HandlerClass())

PythonModelAPI/requirements.txt

@@ -0,0 +1,6 @@
+fastapi
+uvicorn
+pillow
+numpy==1.26.4
+tensorflow==2.8.3
+python-multipart

VisionBuilder.UI.sln

@@ -81,6 +81,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Inspectron.HawkEye", "frame
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Starters", "Starters", "{F3414823-B70E-435E-B4EA-80ABF4371449}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Python", "Python", "{9FA47F4A-8F44-4F55-B6F7-99C962E9F18F}"
+EndProject
+Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "PythonModelAPI", "PythonModelAPI\PythonModelAPI.pyproj", "{B4949493-C2CD-4786-860D-DAACD39A662D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -211,6 +215,8 @@ Global
 		{44D1BA17-FB52-40A2-9D99-E49DA56C10C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{44D1BA17-FB52-40A2-9D99-E49DA56C10C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{44D1BA17-FB52-40A2-9D99-E49DA56C10C2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B4949493-C2CD-4786-860D-DAACD39A662D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B4949493-C2CD-4786-860D-DAACD39A662D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -242,6 +248,7 @@ Global
 		{F7D39916-489A-3583-09A0-175AE82D08B7} = {F2406FBB-DFD3-4CBE-9644-A9FFC2FCBB71}
 		{C55BE2DA-C60B-491C-8668-9517C7F6FF2F} = {D05689E3-04C6-4E3B-ACA7-3F4507CED4CC}
 		{44D1BA17-FB52-40A2-9D99-E49DA56C10C2} = {2B8C63F7-B7FD-464C-9045-180ED8F595F1}
+		{B4949493-C2CD-4786-860D-DAACD39A662D} = {9FA47F4A-8F44-4F55-B6F7-99C962E9F18F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {3CE42AE5-D79F-4E97-A246-AA8FD228B677}