From 28f2f2335852c0d8f109fac5bed9a1df1c6b65a1 Mon Sep 17 00:00:00 2001 From: SimLeek Date: Sun, 3 Mar 2024 17:00:36 -0700 Subject: [PATCH] Added direct display. Removed broken topic code in the displayarray command line file. Made fps_limit match videos by default. Allowed mouse press to help select rectangles. Added bin-packing with p key. Added highlighting selected array. Updated random display example. Made silent operation not require a window. --- displayarray/__init__.py | 1 + displayarray/__main__.py | 31 +----- displayarray/frame/frame_publishing.py | 5 + displayarray/window/__init__.py | 1 + displayarray/window/mglwindow.py | 120 ++++++++++++++++++---- displayarray/window/pyr_quads.frag | 63 +++++++----- displayarray/window/subscriber_windows.py | 12 ++- examples/looping/random_display.py | 14 ++- 8 files changed, 167 insertions(+), 80 deletions(-) diff --git a/displayarray/__init__.py b/displayarray/__init__.py index 15700a7..cf408f0 100644 --- a/displayarray/__init__.py +++ b/displayarray/__init__.py @@ -8,3 +8,4 @@ __version__ = "2.0.0" from .window.subscriber_windows import display, breakpoint_display, read_updates, publish_updates from . import effects +from .window.mglwindow import MglWindow as DirectDisplay \ No newline at end of file diff --git a/displayarray/__main__.py b/displayarray/__main__.py index 9938d50..0620c59 100644 --- a/displayarray/__main__.py +++ b/displayarray/__main__.py @@ -39,38 +39,9 @@ def main(argv=None): v_disps = None if vids: v_disps = display(*vids, blocking=False) - from displayarray.frame.frame_updater import read_updates_ros, read_updates_zero_mq - - topics = arguments["--topic"] - topics_split = [t.split(",") for t in topics] - d = display() - - async def msg_recv(): - nonlocal d - while d: - if arguments["--message-backend"] == "ROS": - async for v_name, frame in read_updates_ros( - [t for t, d in topics_split], [d for t, d in topics_split] - ): - d.update(arr=frame, id=v_name) - if arguments["--message-backend"] == "ZeroMQ": - async for v_name, frame in read_updates_zero_mq( - *[bytes(t, encoding="ascii") for t in topics] - ): - d.update(arr=frame, id=v_name) - - async def update_vids(): while v_disps: - if v_disps: - v_disps.update() - await asyncio.sleep(0) + pass - async def runner(): - await asyncio.wait([msg_recv(), update_vids()]) - - loop = asyncio.get_event_loop() - loop.run_until_complete(runner()) - loop.close() if __name__ == "__main__": diff --git a/displayarray/frame/frame_publishing.py b/displayarray/frame/frame_publishing.py index 122e9eb..717beaa 100644 --- a/displayarray/frame/frame_publishing.py +++ b/displayarray/frame/frame_publishing.py @@ -142,6 +142,11 @@ def pub_cam_loop_opencv( "Only strings or ints representing cameras, or numpy arrays representing pictures supported." ) + if fps_limit == float("inf"): + fps_limit = cam.get(cv2.CAP_PROP_FPS) + if fps_limit is None: + fps_limit = float("inf") + subscriber_dictionary.register_cam(name, cam) frame_counter = 0 diff --git a/displayarray/window/__init__.py b/displayarray/window/__init__.py index 6df33c0..0d10d23 100644 --- a/displayarray/window/__init__.py +++ b/displayarray/window/__init__.py @@ -5,3 +5,4 @@ SubscriberWindows displays one array per window, updating it as it's changed. """ from .subscriber_windows import SubscriberWindows +from .mglwindow import MglWindow \ No newline at end of file diff --git a/displayarray/window/mglwindow.py b/displayarray/window/mglwindow.py index 683cf67..648b409 100644 --- a/displayarray/window/mglwindow.py +++ b/displayarray/window/mglwindow.py @@ -5,6 +5,7 @@ import cv2 import struct from moderngl_window import geometry import os +import rectpack dir_path = os.path.dirname(os.path.realpath(__file__)) @@ -40,6 +41,18 @@ class MglWindowConfig(mgw.WindowConfig): frame = self.hit_buff.hit_level if frame!=-1: self.last_frame = frame + self.uibo.sel_lvl[0] = frame + + def mouse_press_event(self, x, y, button): + # sometimes mouse position event doesn't always trigger, so clicking can now help + if self.uibo is not None: + self.uibo.iMouse[0] = float(x) + self.uibo.iMouse[1] = float(y) + if self.hit_buff is not None: + frame = self.hit_buff.hit_level + if frame != -1: + self.last_frame = frame + self.uibo.sel_lvl[0] = frame def mouse_drag_event(self, x: int, y: int, dx: int, dy: int): if self.hit_buff is not None: @@ -56,7 +69,39 @@ class MglWindowConfig(mgw.WindowConfig): rect[2] += dy rect[3] += dx + def key_event(self, key, action, modifiers): + if key == self.wnd.keys.P and action == self.wnd.keys.ACTION_PRESS: + rects = [] + for i in range(len(self.rbuf.tex_levels)): + r = self.rbuf.tex_levels[i]['rect'] #ltrb + rects.append((r[2]-r[0], r[3]-r[1])) + packer = rectpack.newPacker( + mode=rectpack.PackingMode.Offline, + pack_algo=rectpack.MaxRectsBaf, + bin_algo=rectpack.PackingBin.BFF, + sort_algo=rectpack.SORT_AREA, + rotation=False + ) + bins = [(self.wnd.height, self.wnd.width)] + + for i,r in enumerate(rects): + packer.add_rect(*r, rid=i) + + # Add the bins where the rectangles will be placed + for b in bins: + packer.add_bin(*b) + + # Start packing + packer.pack() + + all_rects = packer.rect_list() + for rect in all_rects: + b, x, y, w, h, rid = rect + self.rbuf.tex_levels[rid]['rect'][0] = x + self.rbuf.tex_levels[rid]['rect'][1] = y + self.rbuf.tex_levels[rid]['rect'][2] = x+w + self.rbuf.tex_levels[rid]['rect'][3] = y + h def create_no_input_texture(width=100, height=100): @@ -78,23 +123,30 @@ class InputTextureInfosUBO(object): def __init__(self, start_textures=[]): self.channels = 3 # Assuming RGB format self.tex_levels = [] - self.input_image: np.ndarray = np.asarray([], dtype=np.float32) + #self.input_image: np.ndarray = np.asarray([], dtype=np.float32) + #self.input_image:bytearray = bytearray() + self.input_image = [] self.no_input = not bool(start_textures) # Initialize input image buffer with a default "no input" image - if not start_textures: - start_textures = [create_no_input_texture()] + #if not start_textures: + # start_textures = [create_no_input_texture()] # Initialize texture levels with default values - for s in start_textures: - tex_level = {'startIdx': 0, 'width': s.shape[0], 'height': s.shape[1], 'flags':0, 'rect': [1.0, 0.0, 0.0, 1.0]} - self.tex_levels.append(tex_level) - self.input_image = np.concatenate((self.input_image, s.flatten()), axis=0, dtype=self.input_image.dtype) + #for s in start_textures: + # tex_level = {'startIdx': 0, 'width': s.shape[0], 'height': s.shape[1], 'flags':0, 'rect': [1.0, 0.0, 0.0, 1.0]} + # self.tex_levels.append(tex_level) + #self.input_image.extend(s.astype(np.float32).tobytes()) + #self.input_image = np.concatenate((self.input_image, s.flatten()), axis=0, dtype=self.input_image.dtype) + # self.input_image.append((s, 0)) def append_input_stream(self, img:np.ndarray, flags:int=0): i = len(self.tex_levels) - start_index = self.tex_levels[-1]['startIdx']+\ - self.tex_levels[-1]['width']*self.tex_levels[-1]['height']*self.channels + if i==0: + start_index = 0 + else: + start_index = self.tex_levels[-1]['startIdx']+\ + self.tex_levels[-1]['width']*self.tex_levels[-1]['height']*self.channels width = img.shape[0] height = img.shape[1] assert img.shape[2] == self.channels @@ -106,15 +158,19 @@ class InputTextureInfosUBO(object): 'flags': flags, 'rect': rect }) - self.input_image = np.concatenate((self.input_image, img.flatten()), axis=0, dtype=self.input_image.dtype) + if isinstance(self.input_image, bytes): + self.input_image = bytearray(self.input_image) + #self.input_image = np.concatenate((self.input_image, img.flatten()), axis=0, dtype=self.input_image.dtype) #self.input_image[start_index:] = img.flatten() + #self.input_image.extend(img.tobytes()) + self.input_image.append((img, start_index)) return i def set_input_stream(self, i, img:np.ndarray, flags:int=0): start_index = self.tex_levels[i]['startIdx'] end_index = start_index + img.shape[0] * img.shape[1] * self.channels - assert img.shape[2] == self.channels + # assert img.shape[2] == self.channels if i!=len(self.tex_levels) and \ self.tex_levels[i]['width']*self.tex_levels[i]['height']!=img.shape[0]*img.shape[1]: ind = start_index @@ -127,7 +183,22 @@ class InputTextureInfosUBO(object): self.tex_levels[i]['height'] = img.shape[1] self.tex_levels[i]['flags'] = flags - self.input_image[start_index:end_index] = img.flatten() + if isinstance(self.input_image, bytearray): + self.input_image = bytes(self.input_image) + + # It seems to be stuck at 200MBps, and this might be a python problem. + # zero-copy would definitely speed things up, but I'm not sure it's possible with OpenCV + # Memcpy should be 10-100 times faster at about 2-20GBps though, + # so if you can access & set the raw data from c++, then that would speed things up 100x + # + # Tried these. Didn't work: + # self.input_image[start_index:end_index] = img.flat + # memoryview(self.input_image)[start_index*4:end_index*4] = memoryview(img.tobytes()) # inpu_image is a bytearray here + # memmove(id(self.input_image)+0x20+start_index*4, id(img.tobytes())+0x20, 4*(end_index-start_index)) + # Mem.view(self.input_image)[start_index*4:end_index*4] = img.data + # an alternative would be to store a list of pointers to img.data or tobytes() and their sizes & offsets, then use write with offset for setting the buffer + #np.copyto(self.input_image[start_index:end_index], img.flat, casting='no') + self.input_image[i] = (img, start_index) def get_tex_data_buffer(self): tex_data_bytes = bytearray() @@ -166,8 +237,11 @@ class InputTextureInfosUBO(object): raise ValueError("Rect must contain 4 values (vec4)") self.tex_levels[level_idx]['rect'] = rect - def get_input_image_buffer(self): - return self.input_image.tobytes() + def get_input_image_buffer(self, writer): + for t in self.input_image: + img, start = t + writer(img.tobytes(), offset=start*4) + #return bytes(self.input_image) def set_input_image_buffer(self, data: np.ndarray): if len(data) != len(self.input_image): @@ -187,10 +261,12 @@ class InputTextureInfosUBO(object): class UserInputUBO: def __init__(self): + self.sel_lvl = np.zeros((1),np.int32) self.iMouse = np.zeros((2,), np.float32) def to_bytes(self): - return struct.pack(f"10000: + # pad = bytes(bytearray([0])*int(-len(buff)%1024)) + # self.input_texture_ubo_buffer.write_chunks(bytes(self.input_texture_infos_ubo.get_input_image_buffer()+pad), 0, 1024, int(np.ceil(len(buff)/1024))) + self.input_texture_infos_ubo.get_input_image_buffer(self.input_texture_ubo_buffer.write) self.input_texture_infos_ubo_buffer.write(self.input_texture_infos_ubo.get_tex_data_buffer()) self.user_input_ubo_buffer.write(self.user_input_ubo.to_bytes()) out_data = self.user_output_ubo_buffer.read() @@ -276,9 +356,10 @@ class MglApp(object): self.update_buffers() self.quad_fs.render(self.shader) +import time class MglWindow(object): - def __init__(self, timer=None, args=None, backend="pygame2"): + def __init__(self, timer=None, args=["--vs=1"], backend="pygame2"): if backend is not None: available = mgw.find_window_classes() assert backend in available, f"backend {backend} is not installed. Installed backends: {available}" @@ -343,6 +424,8 @@ class MglWindow(object): def imshow(self, window_name, frame): if frame.dtype == np.uint8: frame = frame.astype(np.float32) / 255 + elif frame.dtype != np.float32: + frame = frame.astype(np.float32) if window_name in self.window_names.keys(): i = self.window_names[window_name] @@ -359,9 +442,10 @@ class MglWindow(object): # Always bind the window framebuffer before calling render self.window.use() - self.window.render(current_time, delta) if not self.window.is_closing: + self.window.render(current_time, delta) self.window.swap_buffers() + time.sleep(0) else: _, duration = self.timer.stop() self.window.destroy() diff --git a/displayarray/window/pyr_quads.frag b/displayarray/window/pyr_quads.frag index 8f52d85..f65db8d 100644 --- a/displayarray/window/pyr_quads.frag +++ b/displayarray/window/pyr_quads.frag @@ -24,6 +24,7 @@ layout(std430, binding = 1) buffer TexData { }; layout(std430, binding=2) buffer UserInput { + int sel_level; vec2 iMouse; }; @@ -47,26 +48,26 @@ void main() { float x_current = -1; vec2 coord; - for(int i=0;i=texLevels[i].rect.x && - coord.y>=texLevels[i].rect.y && - coord.x= texLevels[i].rect.x && + coord.y >= texLevels[i].rect.y && + coord.x < texLevels[i].rect.z && + coord.y < texLevels[i].rect.w + ) { our_level = i; //don't break. All shader instances should get same execution, and this puts later textures on top. } } - if(our_level!=-1) { - if(bool(texLevels[our_level].flags&TEX_FLAG_HW)){ + if (our_level != -1) { + if (bool(texLevels[our_level].flags & TEX_FLAG_HW)) { coord = gl_FragCoord.yx; - }else{ + } else { coord = gl_FragCoord.xy; } @@ -82,7 +83,7 @@ void main() { int bottomRightIdx = topRightIdx + channels; //leave this for visual debugging - out_color = vec4(float(y_current)/float(levelHeight), float(x_current)/float(levelWidth), 0.0, 1.0); + out_color = vec4(float(y_current) / float(levelHeight), float(x_current) / float(levelWidth), 0.0, 1.0); out_color.x = bilinearInterpolation( fract(x_current), @@ -96,33 +97,49 @@ void main() { out_color.y = bilinearInterpolation( fract(x_current), fract(y_current), - inputImage[bottomLeftIdx+1], - inputImage[bottomRightIdx+1], - inputImage[topLeftIdx+1], - inputImage[topRightIdx+1] + inputImage[bottomLeftIdx + 1], + inputImage[bottomRightIdx + 1], + inputImage[topLeftIdx + 1], + inputImage[topRightIdx + 1] ); } - if(channels>2){ + if (channels > 2) { out_color.z = bilinearInterpolation( fract(x_current), fract(y_current), - inputImage[bottomLeftIdx+2], - inputImage[bottomRightIdx+2], - inputImage[topLeftIdx+2], - inputImage[topRightIdx+2] + inputImage[bottomLeftIdx + 2], + inputImage[bottomRightIdx + 2], + inputImage[topLeftIdx + 2], + inputImage[topRightIdx + 2] ); } - if(bool(texLevels[our_level].flags&TEX_FLAG_BGR)){ + if (bool(texLevels[our_level].flags & TEX_FLAG_BGR)) { float temp_color = out_color.x; out_color.x = out_color.z; out_color.z = temp_color; } // currently only supporting 3 channels at most. - }else{ + } else { // nice white background. ( ∩´ ᐜ `∩) out_color = vec4(1.0, 1.0, 1.0, 1.0); } + if (sel_level != -1) { + if (coord.x >= texLevels[sel_level].rect.x - 1 && + coord.y >= texLevels[sel_level].rect.y - 1 && + coord.x <= texLevels[sel_level].rect.z && + coord.y <= texLevels[sel_level].rect.w + ) { + if (coord.x == texLevels[sel_level].rect.x - 1 || + coord.y == texLevels[sel_level].rect.y - 1 || + coord.x == texLevels[sel_level].rect.z || + coord.y == texLevels[sel_level].rect.w + ) { + out_color = vec4(0.0, 0.5, 0.0, 1.0); // green selection border, on top of everything + } + } + } + if(distance(iMouse, gl_FragCoord.xy)==0){ hit_level = our_level; //hit_pos = vec2(x_current, y_current); diff --git a/displayarray/window/subscriber_windows.py b/displayarray/window/subscriber_windows.py index 59eeee9..f8cb1e8 100644 --- a/displayarray/window/subscriber_windows.py +++ b/displayarray/window/subscriber_windows.py @@ -62,7 +62,7 @@ class SubscriberWindows(object): self.ctx = None self.sock_list: List[zmq.Socket] = [] self.top_list: List[bytes] = [] - self.displayer = mglwindow.MglWindow() + self.displayer = None if callbacks is None: callbacks = [] @@ -70,6 +70,7 @@ class SubscriberWindows(object): self.add_source(name) self.callbacks = callbacks if not self.silent: + self.displayer = mglwindow.MglWindow() for name in window_names: self.add_window(name) @@ -77,10 +78,10 @@ class SubscriberWindows(object): def __bool__(self): self.update() - return not self.exited and not self.displayer.window.is_closing + return not self.exited and (self.displayer is None or not self.displayer.window.is_closing) def __iter__(self): - while not self.exited and not self.displayer.window.is_closing: + while not self.exited and (self.displayer is None or not self.displayer.window.is_closing): self.update() yield self.frames @@ -295,7 +296,8 @@ class SubscriberWindows(object): self.__stop_all_cams() for t in self.close_threads: t.join() - self.displayer.window.close() + if self.displayer is not None: + self.displayer.window.close() def __enter__(self): return self @@ -314,7 +316,7 @@ class SubscriberWindows(object): sub_cmd = window_commands.win_cmd_sub() msg_cmd = "" key = "" - while msg_cmd != "quit" and key != "quit" and (not self.displayer.window.is_closing): + while msg_cmd != "quit" and key != "quit" and (self.displayer is None or not self.displayer.window.is_closing): msg_cmd, key = self.update() sub_cmd.release() window_commands.quit(force_all_read=False) diff --git a/examples/looping/random_display.py b/examples/looping/random_display.py index 2fdd78c..02f9d09 100644 --- a/examples/looping/random_display.py +++ b/examples/looping/random_display.py @@ -1,14 +1,20 @@ from displayarray import display import numpy as np -arr = np.random.normal(0.5, 0.1, (100, 100, 3)) -arr2 = np.random.normal(0.5, 0.1, (200, 200, 3)) -arr3 = np.random.normal(0.5, 0.1, (300, 300, 3)) +arr = np.random.normal(0.5, 0.1, (100, 200, 3)) +arr2 = np.random.normal(0.5, 0.1, (200, 300, 3)) +arr3 = np.random.normal(0.5, 0.1, (300, 400, 3)) with display(arr) as displayer: while displayer: - arr[:] += np.random.normal(0.001, 0.0005, (100, 100, 3)) + arr[:] += np.random.normal(0.001, 0.0005, (100, 200, 3)) arr %= 1.0 + arr2[:] += np.random.normal(0.002, 0.0005, (200, 300, 3)) + arr2 %= 1.0 + arr3[:] -= np.random.normal(0.001, 0.0005, (300, 400, 3)) + arr3 %= 1.0 + + displayer.update(arr2, '2') displayer.update(arr3, '3')