explorer1
Published © GPL3+

Low Power and Encryption enabled Edge AI Camera

Power consumption of Neural Compute Stick is less than 2W. TFLite (3fps) vs OpenVINO (27fps) -> 9X. Encryption enabled to trained models.

IntermediateFull instructions provided6 hours121
Low Power and Encryption enabled Edge AI Camera

Things used in this project

Hardware components

Raspberry Pi 4 Model B
Raspberry Pi 4 Model B
×1
USB-A to Micro-USB Cable
USB-A to Micro-USB Cable
×1

Software apps and online services

OpenVINO™ toolkit
Intel OpenVINO™ toolkit

Story

Read more

Schematics

OpenVINO - Raspberry Pi - NCS2 - Camera module

Code

OpenVINO IR Generation

SH
#-----------------------------------------------------------------------------
# Create FP16 IR files
#-----------------------------------------------------------------------------
!mo.py \
--input_model raw_models/public/mobilenet-ssd/mobilenet-ssd.caffemodel \
--data_type FP16 \
--output_dir models/mobilenet-ssd/FP16 \
--scale 256 \
--mean_values [127,127,127] 

#-----------------------------------------------------------------------------
# Create FP32 IR files
#-----------------------------------------------------------------------------
!mo.py \
--input_model raw_models/public/mobilenet-ssd/mobilenet-ssd.caffemodel \
--data_type FP32 \
--output_dir models/mobilenet-ssd/FP32 \
--scale 256 \
--mean_values [127,127,127]

object_detection.py

Python
#!/usr/bin/env python

from __future__ import print_function
import sys
import os
from argparse import ArgumentParser
import cv2
import time
import logging as log
import numpy as np
import io
from openvino.inference_engine import IECore, IENetwork
from qarpo.demoutils import *
import cv2
import applicationMetricWriter
from qarpo.demoutils import progressUpdate
import base64
import binascii
from cryptography.fernet import Fernet
import argon2

def build_argparser():
    parser = ArgumentParser()
    parser.add_argument('-m', '--model',
                        help='Path to an .xml file with a trained model.',
                        required=True,
                        type=str)
    parser.add_argument('-i', '--input',
                        help='Path to video file or image. \'cam\' for capturing video stream from camera.',
                        required=True,
                        type=str)
    parser.add_argument('-ce', '--cpu_extension',
                        help='MKLDNN-targeted custom layers.'
                             'Absolute path to a shared library with the kernel implementation.',
                        type=str,
                        default=None)
    parser.add_argument('-d', '--device',
                        help='Specify the target device to infer on; CPU, GPU, FPGA, MYRIAD, or HDDL is acceptable.'
                             'Demo will look for a suitable plugin for specified device (CPU by default).',
                        default='CPU',
                        type=str)
    parser.add_argument('-nireq', '--number_infer_requests',
                        help='Number of parallel inference requests (default is 2).',
                        type=int,
                        required=False,
                        default=2)
    parser.add_argument('-s', '--show',
                        help='Show preview to the user.',
                        action='store_true',
                        required=False)
    parser.add_argument('-l', '--labels',
                        help='Labels mapping file.',
                        default=None,
                        type=str)
    parser.add_argument('-pt', '--prob_threshold',
                        help='Probability threshold for detection filtering.',
                        default=0.5,
                        type=float)
    parser.add_argument('-o', '--output_dir',
                        help='Location to store the results of the processing',
                        default=None,
                        required=True,
                        type=str)
    parser.add_argument('-pw', '--password',
                        help='Password used to decrypt .xml&.bin file',
                        default=None,
                        required=False,
                        type=str)
    return parser


def processBoxes(frame_count, res, labels_map, prob_threshold, initial_w, initial_h, result_file):
    for obj in res[0][0]:
        dims = ""
        # Draw only objects when probability more than specified threshold
        if obj[2] > prob_threshold:
            class_id = int(obj[1])
            det_label = labels_map[class_id] if labels_map else "class="+str(class_id)
            dims = "{frame_id} {xmin} {ymin} {xmax} {ymax} {class_id} {det_label} {est} {time} \n".format(frame_id=frame_count, xmin=int(obj[3] * initial_w), ymin=int(obj[4] * initial_h), xmax=int(obj[5] * initial_w), ymax=int(obj[6] * initial_h), class_id=class_id, det_label=det_label, est=round(obj[2]*100, 1), time='N/A')
            result_file.write(dims)

def decrypt(password, in_file):
    with open(in_file, 'r') as f:
        pw_hash = f.readline()[:-1]

    p = argon2.extract_parameters(pw_hash)
    hasher = argon2.PasswordHasher(time_cost=p.time_cost, \
                               memory_cost=p.memory_cost, \
                               parallelism=p.parallelism, \
                               hash_len=p.hash_len, \
                               salt_len=p.salt_len)
    try:
        hasher.verify(pw_hash, password)
        log.info("Argon2 verify: true")
    except:
        log.error("Argon2 verify: false, check password")
        exit()

    salt = pw_hash.split("$")[-2]
    raw_hash = argon2.low_level.hash_secret_raw(time_cost=p.time_cost, \
                                    memory_cost=p.memory_cost, \
                                    parallelism=p.parallelism, \
                                    hash_len=p.hash_len, \
                                    secret=bytes(password, "utf_16_le"), \
                                    salt=bytes(salt, "utf_16_le"), \
                                    type=p.type)

    key = base64.urlsafe_b64encode(raw_hash)

    fernet = Fernet(key)
    dec_data = b''
    with open(in_file, 'rb') as f_in:
        enc_data = f_in.readlines()[1]
        try:
            dec_data = fernet.decrypt(enc_data)
        except:
            log.error("decryption faile")
    return dec_data

def main():
    log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
    args = build_argparser().parse_args()
    model_xml = args.model
    model_bin = os.path.splitext(model_xml)[0] + ".bin"

    # Plugin initialization for specified device and load extensions library if specified
    log.info("Initializing plugin for {} device...".format(args.device))
    ie = IECore()
    if args.cpu_extension and 'CPU' in args.device:
        log.info("Loading plugins for {} device...".format(args.device))
        ie.add_extension(args.cpu_extension, "CPU")

    # Read IR
    log.info("Reading IR...")
    model_encrypted = False
    if args.password != None:
        model_encrypted = True

    if  model_encrypted == False:
        log.info("Model files (.xml &.bin) are un-encrypted")
        net = ie.read_network(model=model_xml, weights=model_bin)
    else:
        #decrypt .xml&.bin files
        log.info("Argon2&Fernet:Model files (.xml &.bin) are encrypted")
        pw = args.password
        dec_xml = decrypt(pw, model_xml)
        dec_bin = decrypt(pw, model_bin)
        net = ie.read_network(model=dec_xml, weights=dec_bin, init_from_buffer=True)      

   
    assert len(net.inputs.keys()) == 1, "Sample supports only single input topologies"
    assert len(net.outputs) == 1, "Sample supports only single output topologies"

    input_blob = next(iter(net.inputs))
    out_blob = next(iter(net.outputs))

    if args.input == 'cam':
        input_stream = 0
    else:
        input_stream = args.input
        assert os.path.isfile(args.input), "Specified input file doesn't exist"

    log.info("Loading IR to the plugin...")
    exec_net = ie.load_network(network=net, num_requests=args.number_infer_requests, device_name=args.device)
 

    log.info("Starting preprocessing...")
    job_id = str(os.environ['PBS_JOBID'])
    result_file = open(os.path.join(args.output_dir, f'output_{job_id}.txt'), "w")
    pre_infer_file = os.path.join(args.output_dir, f'pre_progress_{job_id}.txt')
    infer_file = os.path.join(args.output_dir, f'i_progress_{job_id}.txt')
    processed_vid = '/tmp/processed_vid.bin'

    # Read and pre-process input image
    if isinstance(net.inputs[input_blob], list):
        n, c, h, w = net.inputs[input_blob]
    else:
        n, c, h, w = net.inputs[input_blob].shape
    del net


    cap = cv2.VideoCapture(input_stream)
    video_len = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    if video_len < args.number_infer_requests:
        args.number_infer_requests = video_len 
    #Pre inference processing, read mp4 frame by frame, process using openCV and write to binary file
    width = int(cap.get(3))
    height = int(cap.get(4))
    CHUNKSIZE = n*c*w*h
    id_ = 0
    with open(processed_vid, 'w+b') as f:
        time_start = time.time()
        while cap.isOpened():
            ret, next_frame = cap.read()
            if not ret:
                break
            in_frame = cv2.resize(next_frame, (w, h))
            in_frame = in_frame.transpose((2, 0, 1))  # Change data layout from HWC to CHW
            in_frame = in_frame.reshape((n, c, h, w))
            bin_frame = bytearray(in_frame) 
            f.write(bin_frame)
            id_ += 1
            if id_%10 == 0: 
                progressUpdate(pre_infer_file, time.time()-time_start, id_, video_len) 
    cap.release()

    if args.labels:
        with open(args.labels, 'r') as f:
            labels_map = [x.strip() for x in f]
    else:
        labels_map = None

    log.info("Starting inference in async mode, {} requests in parallel...".format(args.number_infer_requests))
    current_inference = 0
    previous_inference = 1 - args.number_infer_requests
    infer_requests = exec_net.requests
    frame_count = 0

    try:
        infer_time_start = time.time()
        with open(processed_vid, "rb") as data:
            while frame_count < video_len:
                # Read next frame from input stream if available and submit it for inference 
                byte = data.read(CHUNKSIZE)
                if not byte == b"":
                    deserialized_bytes = np.frombuffer(byte, dtype=np.uint8)
                    in_frame = np.reshape(deserialized_bytes, newshape=(n, c, h, w))
                    inf_time = time.time()
                    exec_net.start_async(request_id=current_inference, inputs={input_blob: in_frame})
                
                # Retrieve the output of an earlier inference request
                if previous_inference >= 0:
                    status = infer_requests[previous_inference].wait()
                    if status is not 0:
                        raise Exception("Infer request not completed successfully")
                    # Parse inference results
                    det_time = time.time() - inf_time
                    applicationMetricWriter.send_inference_time(det_time*1000)                      
                    res = infer_requests[previous_inference].outputs[out_blob]
                    processBoxes(frame_count, res, labels_map, args.prob_threshold, width, height, result_file)
                    frame_count += 1

                # Write data to progress tracker
                if frame_count % 10 == 0: 
                    progressUpdate(infer_file, time.time()-infer_time_start, frame_count+1, video_len+1) 

                # Increment counter for the inference queue and roll them over if necessary 
                current_inference += 1
                if current_inference >= args.number_infer_requests:
                    current_inference = 0

                previous_inference += 1
                if previous_inference >= args.number_infer_requests:
                    previous_inference = 0

        # End while loop
        total_time = time.time() - infer_time_start
        with open(os.path.join(args.output_dir, f'stats_{job_id}.txt'), 'w') as f:
                f.write('{:.3g} \n'.format(total_time))
                f.write('{} \n'.format(frame_count))

        result_file.close()
    
    
    finally:
        log.info("Processing done...")
        del exec_net

    # TODO: The model_xml is encrypted so applicationMetricWriter has to be updated 
    # to accept a buffer containing xml instead of file path.
    #applicationMetricWriter.send_application_metrics(model_xml, args.device)

if __name__ == '__main__':
    sys.exit(main() or 0)

Encrypt trained model

Python
import base64
from cryptography.fernet import Fernet
import argon2, binascii

def encrypt(password, argon2_parameters, in_file, out_file):
    hasher = argon2.PasswordHasher(time_cost=argon2_parameters.time_cost, \
                               memory_cost=argon2_parameters.memory_cost, \
                               parallelism=argon2_parameters.parallelism, \
                               hash_len=argon2_parameters.hash_len, \
                               salt_len=argon2_parameters.salt_len)
    pw_hash = hasher.hash(pw)
    salt = pw_hash.split("$")[-2]
    raw_hash = argon2.low_level.hash_secret_raw(time_cost=argon2_parameters.time_cost, \
                                                memory_cost=argon2_parameters.memory_cost, \
                                                parallelism=argon2_parameters.parallelism, \
                                                hash_len=argon2_parameters.hash_len, \
                                                secret=bytes(password, "utf_16_le"), \
                                                salt=bytes(salt, "utf_16_le"), \
                                                type=argon2_parameters.type)
    key = base64.urlsafe_b64encode(raw_hash)
    fernet = Fernet(key)
    with open(in_file, 'rb') as f_in, open(out_file, 'wb') as f_out:
        data = f_in.read()
        enc_data = fernet.encrypt(data)
        pw_hash = pw_hash + '\n'
        f_out.write(bytes(pw_hash, "utf-8"))
        f_out.write(enc_data)

if __name__ == '__main__':
    xml_file = "models/mobilenet-ssd/FP32/mobilenet-ssd.xml"
    bin_file = "models/mobilenet-ssd/FP32/mobilenet-ssd.bin"
    encrypt_xml_file = "models/mobilenet-ssd/FP32/mobilenet-ssd-encrypt.xml"
    encrypt_bin_file = "models/mobilenet-ssd/FP32/mobilenet-ssd-encrypt.bin"

    p = argon2.Parameters(type=argon2.low_level.Type.ID, \
                      version=19, \
                      salt_len=16, \
                      hash_len=32, \
                      time_cost=16, \
                      memory_cost=2**20, \
                      parallelism=8)
    encrypt(pw, p, xml_file, encrypt_xml_file)
    print ("[ok] .xml encrypted")
    encrypt(pw, p, bin_file, encrypt_bin_file)
    print ("[ok] .bin encrypted")

Decrypt model

Python
# IR files decryption function
# input: password and encrypted file path
# output: buffer containing decrypted file content
def decrypt(password, in_file):
    with open(in_file, 'r') as f:
        pw_hash = f.readline()[:-1]

    p = argon2.extract_parameters(pw_hash)
    hasher = argon2.PasswordHasher(time_cost=p.time_cost, \
                               memory_cost=p.memory_cost, \
                               parallelism=p.parallelism, \
                               hash_len=p.hash_len, \
                               salt_len=p.salt_len)
    try:
        hasher.verify(pw_hash, password)
        log.info("Argon2 verify: true")
    except:
        log.error("Argon2 verify: false, check password")
        exit()
    salt = pw_hash.split("$")[-2]
    raw_hash = argon2.low_level.hash_secret_raw(time_cost=p.time_cost, \
                                    memory_cost=p.memory_cost, \
                                    parallelism=p.parallelism, \
                                    hash_len=p.hash_len, \
                                    secret=bytes(password, "utf_16_le"), \
                                    salt=bytes(salt, "utf_16_le"), \
                                    type=p.type)

    key = base64.urlsafe_b64encode(raw_hash)
    fernet = Fernet(key)
    dec_data = b''
    with open(in_file, 'rb') as f_in:
        enc_data = f_in.readlines()[1]
        try:
            dec_data = fernet.decrypt(enc_data)
        except:
            log.error("decryption faile")
    return dec_data

#------------------------------------------------------------------------
...
# decrypting and loading the model IR files
dec_xml = decrypt(pw, model_xml)
dec_bin = decrypt(pw, model_bin)
net = ie.read_network(model=dec_xml, weights=dec_bin, init_from_buffer=True)
...

Credits

explorer1

explorer1

2 projects • 0 followers
Thanks to OpenVINO.

Comments

Related channels and tags