pyTorch模型部署--高并发web服务c++移动端ncnn

import asyncioimport itertoolsimport functoolsfrom sanic import Sanicfrom se import json, textfrom import loggerfrom ions import ServerErrorimport sanicimport threadingimport mport ioimport torchimport torchvision# from cyclegan import get_pretrained_modelapp = Sanic(__name__)device = ('cpu')# we only run 1 inference run at any time (one could schedule between several runners if desired)MAX_QUEUE_SIZE = 3 # we accept a backlog of MAX_QUEUE_SIZE before handing out "Too busy" errorsMAX_BATCH_SIZE = 2 # we put at most MAX_BATCH_SIZE things in a single batchMAX_WAIT = 1 # we wait at most MAX_WAIT seconds before running for more inputs to arrive in batchingclass HandlingError(Exception): def __init__(self, msg, code=500): super().__init__() ng_code = code ng_msg = msgclass ModelRunner: def __init__(self):#, model_name # _name = model_name = [] _lock = None # = get_pretrained_model(_name, # map_location=device) = 18(pretrained=True) () for p in ters(): es_grad_(False) _processing = None _processing_timer = None def schedule_processing_if_needed(self): if len() >= MAX_BATCH_SIZE: ("next batch ready when processing a batch") _() elif : ("queue nonempty when processing a batch, setting next timer") _processing_timer = _at([0]["time"] + MAX_WAIT, _) async def process_input(self, input): our_task = {"done_event": (loop=), "input": input, "time": ()} async with _lock: if len() >= MAX_QUEUE_SIZE: raise HandlingError("I'm too busy", code=503) (our_task)

("enqueued task. new queue size {}".format(len())) le_processing_if_needed() await our_task["done_event"].wait() return our_task["output"] def run_model(self, batch): # runs in other thread return ((device)).to('cpu') async def model_runner(self): _lock = (loop=) _processing = (loop=) # ("started model runner for {}".format(_name)) while True: await _() _() if _processing_timer is not None: _processing_() _processing_timer = None async with _lock: if : longest_wait = () - [0]["time"] else: # oops longest_wait = None ("launching processing. queue size: {}. longest wait: {}".format(len(), longest_wait)) to_process = [:MAX_BATCH_SIZE] del [:len(to_process)] le_processing_if_needed() # so here we copy, it would be neater to avoid this batch = ([t["input"] for t in to_process], dim=0) # we could delete result = await _in_executor( None, l(_model, batch) ) for t, r in zip(to_process, result): t["output"] = r t["done_event"].set() del to_processstyle_transfer_runner = ModelRunner()#[1]@('/image', methods=['POST'], stream=True)async def image(request): try: print (s) content_length = int(('content-length', '0')) MAX_SIZE = 2**22 # 10MB if content_length: if content_length > MAX_SIZE: raise HandlingError("Too large") data = bytearray(content_length) else: data = bytearray(MAX_SIZE) pos = 0 while True: # so this still copies too much stuff. data_part = await () if data_part is None: break data[pos: len(data_part) + pos] = data_part pos += len(data_part) if pos > MAX_SIZE: raise HandlingError("Too large") # ideally, we would

"-lopencv_cudaimgproc", "-lopencv_cudalegacy", "-lopencv_cudaobjdetect", "-lopencv_cudaoptflow", "-lopencv_cudastereo", "-lopencv_cudawarping", "-lopencv_cudev", "-lopencv_datasets", "-lopencv_dnn_objdetect", "-lopencv_dnn", "-lopencv_dpm", "-lopencv_face", "-lopencv_features2d", "-lopencv_flann", "-lopencv_freetype", "-lopencv_fuzzy", "-lopencv_gapi", "-lopencv_hdf", "-lopencv_hfs", "-lopencv_highgui", "-lopencv_imgcodecs", "-lopencv_img_hash", "-lopencv_imgproc", "-lopencv_line_descriptor", "-lopencv_ml", "-lopencv_objdetect", "-lopencv_optflow", "-lopencv_phase_unwrapping", "-lopencv_photo", "-lopencv_plot", "-lopencv_quality", "-lopencv_reg", "-lopencv_rgbd", "-lopencv_saliency", "-lopencv_shape", "-lopencv_stereo", "-lopencv_stitching", "-lopencv_structured_light", "-lopencv_superres", "-lopencv_surface_matching", "-lopencv_text", "-lopencv_tracking", "-lopencv_videoio", "-lopencv_video", "-lopencv_videostab", "-lopencv_xfeatures2d", "-lopencv_ximgproc", "-lopencv_xobjdetect", "-lopencv_xphoto", "-L/usr/lib/x86_64-linux-gnu", "-lvulkan", "-L${workspaceFolder}/../build/glslang/glslang", "-lglslang", "-L${workspaceFolder}/../build/glslang/glslang/OSDependent/Unix", "-lOSDependent", "-L${workspaceFolder}/../build/glslang/OGLCompilersDLL", "-lOGLCompiler", "-L${workspaceFolder}/../build/glslang/SPIRV", "-lSPIRV" ], "options": { "cwd": "${workspaceFolder}" }, "problemMatcher": [ "$gcc" ],

], "group": { "kind": "build", "isDefault": true } } ], "version": "2.0.0"}关键点："-std=c++11", "-fopenmp",-lncnn,-lvulkan,-lglslang,-lOSDependent,-lOGLCompiler,-lSPIRV;4.2 pyTorch mobile