Team SneakyTurtle Submission - Updated deployment

.gitignore

@@ -0,0 +1 @@
+.env

Deployment/app.py

@@ -0,0 +1,19 @@
+from flask import Flask, render_template, request, url_for
+from prediction import predict
+
+app = Flask(__name__, template_folder = "templates")
+
+
+# Controllers
+@app.route("/", methods = ["GET", "POST"])
+def home():
+    output = "hold"
+    if request.method == "POST":
+        f = request.files['file']
+        f.save("inputImage.jpg") 
+        output = "show"
+        predict()
+    return render_template("index.html", output = output)
+
+if __name__ == "__main__":
+    app.run(debug = True)

Deployment/prediction.py

@@ -0,0 +1,123 @@
+import cv2
+import supervision as sv
+import torch
+import pytorch_lightning as pl
+from transformers import DetrForObjectDetection
+
+
+class Detr(pl.LightningModule):
+
+    def __init__(self, lr, lr_backbone, weight_decay):
+        super().__init__()
+        self.model = DetrForObjectDetection.from_pretrained(
+            pretrained_model_name_or_path= "Deployment/ObjectDetectionModel-state.pth", 
+            num_labels=len(id2label),
+            ignore_mismatched_sizes=True
+        )
+
+        self.lr = lr
+        self.lr_backbone = lr_backbone
+        self.weight_decay = weight_decay
+
+    def forward(self, pixel_values, pixel_mask):
+        return self.model(pixel_values=pixel_values, pixel_mask=pixel_mask)
+
+    def common_step(self, batch, batch_idx):
+        pixel_values = batch["pixel_values"]
+        pixel_mask = batch["pixel_mask"]
+        labels = [{k: v.to(self.device) for k, v in t.items()} for t in batch["labels"]]
+
+        outputs = self.model(pixel_values=pixel_values, pixel_mask=pixel_mask, labels=labels)
+
+        loss = outputs.loss
+        loss_dict = outputs.loss_dict
+
+        return loss, loss_dict
+
+    def training_step(self, batch, batch_idx):
+        loss, loss_dict = self.common_step(batch, batch_idx)     
+        # logs metrics for each training_step, and the average across the epoch
+        self.log("training_loss", loss)
+        for k,v in loss_dict.items():
+            self.log("train_" + k, v.item())
+
+        return loss
+
+    def validation_step(self, batch, batch_idx):
+        loss, loss_dict = self.common_step(batch, batch_idx)     
+        self.log("validation/loss", loss)
+        for k, v in loss_dict.items():
+            self.log("validation_" + k, v.item())
+
+        return loss
+
+    def configure_optimizers(self):
+        param_dicts = [
+            {
+                "params": [p for n, p in self.named_parameters() if "backbone" not in n and p.requires_grad]},
+            {
+                "params": [p for n, p in self.named_parameters() if "backbone" in n and p.requires_grad],
+                "lr": self.lr_backbone,
+            },
+        ]
+        return torch.optim.AdamW(param_dicts, lr=self.lr, weight_decay=self.weight_decay)
+
+
+DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+CHECKPOINT = 'facebook/detr-resnet-50'
+CONFIDENCE_TRESHOLD = 0.5
+IOU_TRESHOLD = 0.8
+
+categories = {0: {'id': 0, 'name': 'cars-trafficlights-pedestrian-', 'supercategory': 'none'},
+ 1: {'id': 1, 'name': 'biker', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 2: {'id': 2, 'name': 'car', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 3: {'id': 3, 'name': 'pedestrian', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 4: {'id': 4, 'name': 'trafficLight', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 5: {'id': 5, 'name': 'trafficLight-Green', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 6: {'id': 6, 'name': 'trafficLight-GreenLeft', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 7: {'id': 7, 'name': 'trafficLight-Red', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 8: {'id': 8, 'name': 'trafficLight-RedLeft', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 9: {'id': 9, 'name': 'trafficLight-Yellow', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 10: {'id': 10, 'name': 'trafficLight-YellowLeft', 'supercategory': 'cars-trafficlights-pedestrian-'},
+ 11: {'id': 11, 'name': 'truck', 'supercategory': 'cars-trafficlights-pedestrian-'}}
+
+id2label = {k: v['name'] for k,v in categories.items()}
+box_annotator = sv.BoxAnnotator()
+
+
+
+image_path = "static/inputImage.jpg"
+image = cv2.imread(image_path)
+
+
+
+model = Detr(lr=1e-4, lr_backbone=1e-5, weight_decay=1e-4)
+model.load_state_dict(torch.load("ObjectDetectionModel-state"))
+model.to(DEVICE)
+
+
+
+
+def predict():
+    # inference
+    with torch.no_grad():
+
+        # load image and predict
+        inputs = image_processor(images=image, return_tensors='pt').to(DEVICE)
+        outputs = model(**inputs)
+
+        # post-process
+        target_sizes = torch.tensor([image.shape[:2]]).to(DEVICE)
+        results = image_processor.post_process_object_detection(
+            outputs=outputs, 
+            threshold=CONFIDENCE_TRESHOLD, 
+            target_sizes=target_sizes
+        )[0]
+
+    # annotate
+    detections = sv.Detections.from_transformers(transformers_results=results).with_nms(threshold=0.5)
+    labels = [f"{id2label[class_id]} {confidence:.2f}" for _, confidence, class_id, _ in detections]
+    frame = box_annotator.annotate(scene=image.copy(), detections=detections, labels=labels)
+
+
+    cv2.imsave(frame, "static/output.jpg")

Deployment/requirements.txt

@@ -0,0 +1,64 @@
+aiohttp==3.8.4
+aiosignal==1.3.1
+async-timeout==4.0.2
+attrs==23.1.0
+blinker==1.6.2
+certifi==2023.5.7
+charset-normalizer==3.1.0
+click==8.1.3
+colorama==0.4.6
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+dill==0.3.6
+filelock==3.12.0
+Flask==2.3.2
+fonttools==4.39.4
+frozenlist==1.3.3
+fsspec==2023.5.0
+huggingface-hub==0.15.1
+idna==3.4
+importlib-metadata==6.6.0
+importlib-resources==5.12.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+kiwisolver==1.4.4
+lightning-utilities==0.8.0
+MarkupSafe==2.1.3
+matplotlib==3.7.1
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+networkx==3.1
+numpy==1.24.3
+opencv-python==4.7.0.72
+packaging==23.1
+pandas==2.0.2
+Pillow==9.5.0
+pyarrow==12.0.0
+pyparsing==3.0.9
+python-dateutil==2.8.2
+pytorch-lightning==2.0.3
+pytz==2023.3
+PyYAML==6.0
+regex==2023.6.3
+requests==2.31.0
+responses==0.18.0
+safetensors==0.3.1
+six==1.16.0
+supervision==0.9.0
+sympy==1.12
+timm==0.9.2
+tokenizers==0.13.3
+torch==2.0.1
+torchmetrics==0.11.4
+torchvision==0.15.2
+tqdm==4.65.0
+transformers==4.30.0
+typing_extensions==4.6.3
+tzdata==2023.3
+urllib3==2.0.3
+Werkzeug==2.3.5
+xxhash==3.2.0
+yarl==1.9.2
+zipp==3.15.0

Deployment/templates/index.html

@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-9ndCyUaIbzAi2FUVXJi0CjmCapSmO7SnpJef0486qhLnuZ2cdeRhO02iuK6FUUVM" crossorigin="anonymous">
+    <title>Object Detection</title>
+</head>
+<body>
+    <form method = "POST" enctype="multipart/form-data">
+
+        <div class="app" style="margin: auto;width: 50%; padding-top: 12.5%;">
+
+            <h1>Object Detection for for Autonomous Vehicles</h1>
+            <h3>Team: Sneakyturtle</h3>
+
+            <div class="input-group mb-3">
+                <input type="file" class="form-control" id="inputGroupFile02">
+                <label class="input-group-text" for="inputGroupFile02">Upload</label>
+            </div>
+
+            <button type="submit" class="btn btn-success">Submit</button>
+
+
+            {% if output == "show" %}
+
+            <div class="img" style = "padding-bottom:5%;">
+                <h6>Output</h6>
+                <img src="{{url_for('static', filename = 'output.png')}}" width="500" height="500" />
+            </div>
+            {% endif %}
+
+        </div>
+
+
+    </form>
+</body>
+</html>

README.md

@@ -1,17 +1,111 @@
 # intel-oneAPI
 
-#### Team Name -
-#### Problem Statement - 
-#### Team Leader Email -
+#### Team Name - SneakyTurtle <img src = "https://github.com/SneakyTurtIe/intel-oneAPI/assets/59119736/f7ee32ba-d85e-4ab3-8731-b6701d94b313" height = "50px">
 
-## A Brief of the Prototype:
-  This section must include UML Daigrms and prototype description
+#### Problem Statement - Object Detection For Autonomous Vehicles <img src = "https://github.com/SneakyTurtIe/intel-oneAPI/assets/59119736/549af82b-79e6-436a-b067-2e83641a9095" height = "50px">
+#### Team Leader Email - [email protected]
+
+
+
+
+## <img src = "https://github.com/SneakyTurtIe/intel-oneAPI/assets/59119736/80522c5e-f814-4c3d-906a-e5518ccdd150" height = "50px" > A Brief of the Prototype:
+
+
+  <img src ="https://media.giphy.com/media/dqQxplKANtn8c/giphy.gif" height = "250" width = "100%"/>
 
-## Tech Stack: 
-   List Down all technologies used to Build the prototype **Clearly mentioning Intel® AI Analytics Toolkits, it's libraries and the SYCL/DCP++ Libraries used**
-
-## Step-by-Step Code Execution Instructions:
-  This Section must contain set of instructions required to clone and run the prototype, so that it can be tested and deeply analysed
 
+This repository contains the code and resources for an advanced object detection model for autonomous vehicles. The Prototpe is designed to excel in extreme weather conditions like fog and snow, thereby improving the safety and reliability of autonomous driving systems.
+
+### Dataset Acquisition
+The dataset used in this project was obtained from Roboflow, a platform for computer vision data management. The dataset can be accessed using the following link: [Roboflow - Self-driving Car Object Detection](https://public.roboflow.com/object-detection/self-driving-car)
+
+### Data Preprocessing
+To prepare the dataset for training, we utilized the preprocessing capabilities offered by Roboflow. The following preprocessing steps were applied to the images:
+
+- Auto-orientation of pixel data, including EXIF-orientation stripping.
+- Resizing the images to a resolution of 640x640 pixels using a stretch method.
+- Random brightness adjustment of the images within a range of -26% to +26%.
+- Random Gaussian blur with a variance between 0 and 2 pixels.
+- The application of salt and pepper noise to 2% of the pixels in the images.
+
+### Model Training
+After preprocessing the dataset, we employed the DETR (Detection Transformer) transfer learning technique to train our advanced object detection model. By leveraging this state-of-the-art approach, we aimed to enhance the model's ability to accurately detect and classify objects in real-time, even in challenging weather conditions.
+
+### Deployment
+For the deployment of the trained prototype, we used Flask. We developed a web application that provides an intuitive interface for users to use the prototype. The Flask app allows users to upload images for object detection. The detected objects are then visualized and displayed to the user, providing valuable insights and enhancing the capabilities of autonomous driving systems.
+
+
+<p align = "center" > 
+  <img src = "https://github.com/SneakyTurtIe/intel-oneAPI/assets/59119736/114c452f-7a70-4b0c-9ba8-96bb1857bdbe">
+</p>
+
+
+### Process Flow Diagram
+![image](https://github.com/SneakyTurtIe/intel-oneAPI/assets/59119736/8116caf3-48df-4d64-9c89-39b222329b5f)
+
+
+### Architecture Diagram
+![image](https://github.com/SneakyTurtIe/intel-oneAPI/assets/59119736/4ae5024a-068c-42aa-abca-6adfab25bb46)
+
+
+
+### Repository Structure
+- `Deployment/`: You can find the deployment of the prototype.
+- `notebooks/`: Jupyter notebooks showcasing the data preprocessing, model training, and evaluation processes.
+- `README.md`: The file you are currently reading, providing an overview of the project.
+
+
+**Note:** Please download the model file from [here](https://drive.google.com/file/d/1rD6eBrp7F3GFdWe_0e8SaDgmxEwCekhQ/view?usp=sharing) and save it in the `Deployment/` repository before use.
+
+
+
+##  <img src = "https://github.com/SneakyTurtIe/intel-oneAPI/assets/59119736/51de0cd9-f6e8-43ce-bb09-baccd71c0447" height = "50px">  Tech Stack: 
+1. Roboflow: For image input and preprocessing.
+1. OneDAN: For data analysis of images and drawing conclusions.
+1. OneDNN: for framework optimization.
+1. OneVPL: video processing tasks.
+1. Google Colaboratory: For coding and effective communication between team members.
+1. Pytorch framework
+1. Flask: For deployment of the solution.
+
+
+![NumPy](https://img.shields.io/badge/numpy-%23013243.svg?style=for-the-badge&logo=numpy&logoColor=white)
+![Pandas](https://img.shields.io/badge/pandas-%23150458.svg?style=for-the-badge&logo=pandas&logoColor=white)
+![PyTorch](https://img.shields.io/badge/PyTorch-%23EE4C2C.svg?style=for-the-badge&logo=PyTorch&logoColor=white)
+![Colab](https://img.shields.io/badge/google%20colab-%040404.svg?style=for-the-badge&logo=googlecolab&logoColor=white)
+![Flask](https://img.shields.io/badge/flask-%23000.svg?style=for-the-badge&logo=flask&logoColor=white)  
+and more..
+
+## <img src = "https://github.com/SneakyTurtIe/intel-oneAPI/assets/59119736/8d918e92-7fa3-4218-80b9-d47a0c0e7bcb" height = "50px"> Step-by-Step Code Execution Instructions:
+  #### To clone and run the prototype use the following instructions:
+  1. Open up a bash terminal(Linux) or Command Prompt(Windows)
+  2. Cone the repository with the code: `git clone https://github.com/SneakyTurtIe/intel-oneAPI.git`
+  3. Change the working directory to the deployment directory: `cd intel-oneAPI/Deployment`
+  4. Install the requirements of the project with `pip install -r requirements.txt`
+  5. Run the deployment on localhost using the command: `python app.py`
+
+<p align="center">
+  Your prototype should be running at localhost:5000 <br>
+  <img src = "https://media.giphy.com/media/JqDeI2yjpSRgdh35oe/giphy.gif" height = "250"/>
+</p>
+
 ## What I Learned:
-   Write about the biggest learning you had while developing the prototype
+During the course of our project in object detection for autonomous vehicles, we have gained valuable insights and experiences. Here are the elaborations and additional points:
+
+Limitations of existing models: We have identified several limitations in existing object detection models, particularly in extreme weather conditions such as fog, mist, and camouflage. These conditions can significantly affect the accuracy and reliability of object detection algorithms, making it essential to address these challenges.
+
+Specialized models for extreme weather: Recognizing the limitations, we have emphasized the importance of developing specialized models specifically tailored for extreme weather conditions. These models incorporate advanced techniques and algorithms that can handle the challenges posed by weather-related factors, enabling more accurate and robust object detection.
+
+Data augmentation for weather conditions: To tackle the limitations caused by extreme weather, we performed data augmentation techniques. By augmenting the dataset to resemble various weather conditions, such as fog and mist, we aimed to train the object detection models to be more resilient and adaptable in adverse weather scenarios.
+
+Pre-processing steps: As part of the data preparation, we applied pre-processing steps to each image. These steps included auto-orientation of pixel data with EXIF-orientation stripping and resizing the images to a standardized size of 640x640 pixels, allowing consistent input for the object detection models.
+
+Augmentation techniques: To create diverse variations of each source image, we employed augmentation techniques. This involved randomly adjusting the brightness of the images within a range of -26% to +26%, applying random Gaussian blur with a range of 0 to 2 pixels, and introducing salt and pepper noise to 2% of the pixels. These augmentations aimed to increase the variability of the dataset and improve the model's generalization capabilities.
+
+Leveraging Intel's oneAPI OneDNN: Throughout our project, we explored the capabilities of Intel's oneAPI OneDNN tool. By utilizing this deep neural network tool, we were able to optimize and accelerate the performance of our object detection code. Leveraging hardware accelerators and parallel computing, we achieved improved efficiency and speed in our models.
+
+Overall, our project focused on addressing the limitations of existing object detection models in extreme weather conditions. Through specialized model development, data augmentation, and the use of advanced tools like Intel's oneAPI OneDNN, we aimed to improve the accuracy, reliability, and robustness of object detection for autonomous vehicles in challenging weather scenarios.
+
+
+
+