Added direct display. Removed broken topic code in the displayarray command line file. Made fps_limit match videos by default. Allowed mouse press to help select rectangles. Added bin-packing with p key. Added highlighting selected array. Updated random display example. Made silent operation not require a window.

2024-03-03 17:00:36 -07:00
parent b86e150cd0
commit 28f2f23358
8 changed files with 167 additions and 80 deletions
@@ -8,3 +8,4 @@ __version__ = "2.0.0"

 from .window.subscriber_windows import display, breakpoint_display, read_updates, publish_updates
 from . import effects
+from .window.mglwindow import MglWindow as DirectDisplay
@@ -39,38 +39,9 @@ def main(argv=None):
    v_disps = None
    if vids:
        v_disps = display(*vids, blocking=False)
-    from displayarray.frame.frame_updater import read_updates_ros, read_updates_zero_mq
-
-    topics = arguments["--topic"]
-    topics_split = [t.split(",") for t in topics]
-    d = display()
-
-    async def msg_recv():
-        nonlocal d
-        while d:
-            if arguments["--message-backend"] == "ROS":
-                async for v_name, frame in read_updates_ros(
-                    [t for t, d in topics_split], [d for t, d in topics_split]
-                ):
-                    d.update(arr=frame, id=v_name)
-            if arguments["--message-backend"] == "ZeroMQ":
-                async for v_name, frame in read_updates_zero_mq(
-                    *[bytes(t, encoding="ascii") for t in topics]
-                ):
-                    d.update(arr=frame, id=v_name)
-
-    async def update_vids():
        while v_disps:
-            if v_disps:
-                v_disps.update()
-                await asyncio.sleep(0)
+            pass

-    async def runner():
-        await asyncio.wait([msg_recv(), update_vids()])
-
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(runner())
-    loop.close()


 if __name__ == "__main__":
@@ -142,6 +142,11 @@ def pub_cam_loop_opencv(
            "Only strings or ints representing cameras, or numpy arrays representing pictures supported."
        )

+    if fps_limit == float("inf"):
+        fps_limit = cam.get(cv2.CAP_PROP_FPS)
+        if fps_limit is None:
+            fps_limit = float("inf")
+
    subscriber_dictionary.register_cam(name, cam)

    frame_counter = 0
@@ -5,3 +5,4 @@ SubscriberWindows displays one array per window, updating it as it's changed.
 """

 from .subscriber_windows import SubscriberWindows
+from .mglwindow import MglWindow
@@ -5,6 +5,7 @@ import cv2
 import struct
 from moderngl_window import geometry
 import os
+import rectpack

 dir_path = os.path.dirname(os.path.realpath(__file__))

@@ -40,6 +41,18 @@ class MglWindowConfig(mgw.WindowConfig):
            frame = self.hit_buff.hit_level
            if frame!=-1:
                self.last_frame = frame
+                self.uibo.sel_lvl[0] = frame
+
+    def mouse_press_event(self, x, y, button):
+        # sometimes mouse position event doesn't always trigger, so clicking can now help
+        if self.uibo is not None:
+            self.uibo.iMouse[0] = float(x)
+            self.uibo.iMouse[1] = float(y)
+        if self.hit_buff is not None:
+            frame = self.hit_buff.hit_level
+            if frame != -1:
+                self.last_frame = frame
+                self.uibo.sel_lvl[0] = frame

    def mouse_drag_event(self, x: int, y: int, dx: int, dy: int):
        if self.hit_buff is not None:
@@ -56,7 +69,39 @@ class MglWindowConfig(mgw.WindowConfig):
                rect[2] += dy
                rect[3] += dx

+    def key_event(self, key, action, modifiers):
+        if key == self.wnd.keys.P and action == self.wnd.keys.ACTION_PRESS:
+            rects = []
+            for i in range(len(self.rbuf.tex_levels)):
+                r = self.rbuf.tex_levels[i]['rect']  #ltrb
+                rects.append((r[2]-r[0], r[3]-r[1]))
+            packer = rectpack.newPacker(
+                mode=rectpack.PackingMode.Offline,
+                pack_algo=rectpack.MaxRectsBaf,
+                bin_algo=rectpack.PackingBin.BFF,
+                sort_algo=rectpack.SORT_AREA,
+                rotation=False
+            )

+            bins = [(self.wnd.height, self.wnd.width)]
+
+            for i,r in enumerate(rects):
+                packer.add_rect(*r, rid=i)
+
+            # Add the bins where the rectangles will be placed
+            for b in bins:
+                packer.add_bin(*b)
+
+            # Start packing
+            packer.pack()
+
+            all_rects = packer.rect_list()
+            for rect in all_rects:
+                b, x, y, w, h, rid = rect
+                self.rbuf.tex_levels[rid]['rect'][0] = x
+                self.rbuf.tex_levels[rid]['rect'][1] = y
+                self.rbuf.tex_levels[rid]['rect'][2] = x+w
+                self.rbuf.tex_levels[rid]['rect'][3] = y + h


 def create_no_input_texture(width=100, height=100):
@@ -78,23 +123,30 @@ class InputTextureInfosUBO(object):
    def __init__(self, start_textures=[]):
        self.channels = 3  # Assuming RGB format
        self.tex_levels = []
-        self.input_image: np.ndarray = np.asarray([], dtype=np.float32)
+        #self.input_image: np.ndarray = np.asarray([], dtype=np.float32)
+        #self.input_image:bytearray = bytearray()
+        self.input_image = []
        self.no_input = not bool(start_textures)

        # Initialize input image buffer with a default "no input" image
-        if not start_textures:
-            start_textures = [create_no_input_texture()]
+        #if not start_textures:
+        #    start_textures = [create_no_input_texture()]

        # Initialize texture levels with default values
-        for s in start_textures:
-            tex_level = {'startIdx': 0, 'width': s.shape[0], 'height': s.shape[1], 'flags':0, 'rect': [1.0, 0.0, 0.0, 1.0]}
-            self.tex_levels.append(tex_level)
-            self.input_image = np.concatenate((self.input_image, s.flatten()), axis=0, dtype=self.input_image.dtype)
+        #for s in start_textures:
+        #    tex_level = {'startIdx': 0, 'width': s.shape[0], 'height': s.shape[1], 'flags':0, 'rect': [1.0, 0.0, 0.0, 1.0]}
+        #    self.tex_levels.append(tex_level)
+            #self.input_image.extend(s.astype(np.float32).tobytes())
+            #self.input_image = np.concatenate((self.input_image, s.flatten()), axis=0, dtype=self.input_image.dtype)
+        #    self.input_image.append((s, 0))

    def append_input_stream(self, img:np.ndarray, flags:int=0):
        i = len(self.tex_levels)
-        start_index = self.tex_levels[-1]['startIdx']+\
-                      self.tex_levels[-1]['width']*self.tex_levels[-1]['height']*self.channels
+        if i==0:
+            start_index = 0
+        else:
+            start_index = self.tex_levels[-1]['startIdx']+\
+                          self.tex_levels[-1]['width']*self.tex_levels[-1]['height']*self.channels
        width = img.shape[0]
        height = img.shape[1]
        assert img.shape[2] == self.channels
@@ -106,15 +158,19 @@ class InputTextureInfosUBO(object):
            'flags': flags,
            'rect': rect
        })
-        self.input_image = np.concatenate((self.input_image, img.flatten()), axis=0, dtype=self.input_image.dtype)
+        if isinstance(self.input_image, bytes):
+            self.input_image = bytearray(self.input_image)
+        #self.input_image = np.concatenate((self.input_image, img.flatten()), axis=0, dtype=self.input_image.dtype)
        #self.input_image[start_index:] =  img.flatten()
+        #self.input_image.extend(img.tobytes())
+        self.input_image.append((img, start_index))

        return i

    def set_input_stream(self, i, img:np.ndarray, flags:int=0):
        start_index = self.tex_levels[i]['startIdx']
        end_index = start_index + img.shape[0] * img.shape[1] * self.channels
-        assert img.shape[2] == self.channels
+        # assert img.shape[2] == self.channels
        if i!=len(self.tex_levels) and \
            self.tex_levels[i]['width']*self.tex_levels[i]['height']!=img.shape[0]*img.shape[1]:
            ind = start_index
@@ -127,7 +183,22 @@ class InputTextureInfosUBO(object):
        self.tex_levels[i]['height'] = img.shape[1]
        self.tex_levels[i]['flags'] = flags

-        self.input_image[start_index:end_index] = img.flatten()
+        if isinstance(self.input_image, bytearray):
+            self.input_image = bytes(self.input_image)
+
+        # It seems to be stuck at 200MBps, and this might be a python problem.
+        # zero-copy would definitely speed things up, but I'm not sure it's possible with OpenCV
+        # Memcpy should be 10-100 times faster at about 2-20GBps though,
+        # so if you can access & set the raw data from c++, then that would speed things up 100x
+        #
+        # Tried these. Didn't work:
+        #     self.input_image[start_index:end_index] = img.flat
+        #     memoryview(self.input_image)[start_index*4:end_index*4] = memoryview(img.tobytes())  # inpu_image is a bytearray here
+        #     memmove(id(self.input_image)+0x20+start_index*4, id(img.tobytes())+0x20, 4*(end_index-start_index))
+        #     Mem.view(self.input_image)[start_index*4:end_index*4] = img.data
+        # an alternative would be to store a list of pointers to img.data or tobytes() and their sizes & offsets, then use write with offset for setting the buffer
+        #np.copyto(self.input_image[start_index:end_index], img.flat, casting='no')
+        self.input_image[i] = (img, start_index)

    def get_tex_data_buffer(self):
        tex_data_bytes = bytearray()
@@ -166,8 +237,11 @@ class InputTextureInfosUBO(object):
            raise ValueError("Rect must contain 4 values (vec4)")
        self.tex_levels[level_idx]['rect'] = rect

-    def get_input_image_buffer(self):
-        return self.input_image.tobytes()
+    def get_input_image_buffer(self, writer):
+        for t in self.input_image:
+            img, start = t
+            writer(img.tobytes(), offset=start*4)
+        #return bytes(self.input_image)

    def set_input_image_buffer(self, data: np.ndarray):
        if len(data) != len(self.input_image):
@@ -187,10 +261,12 @@ class InputTextureInfosUBO(object):

 class UserInputUBO:
    def __init__(self):
+        self.sel_lvl = np.zeros((1),np.int32)
        self.iMouse = np.zeros((2,), np.float32)

    def to_bytes(self):
-        return struct.pack(f"<ff",
+        return struct.pack(f"<ixxxxff",
+                           *self.sel_lvl,
                           *self.iMouse)

    @property
@@ -260,7 +336,11 @@ class MglApp(object):

    def update_buffers(self):
        # Update uniform buffers
-        self.input_texture_ubo_buffer.write(self.input_texture_infos_ubo.get_input_image_buffer())
+        #buff = self.input_texture_infos_ubo.get_input_image_buffer()
+        #if len(buff)>10000:
+        #    pad = bytes(bytearray([0])*int(-len(buff)%1024))
+        #    self.input_texture_ubo_buffer.write_chunks(bytes(self.input_texture_infos_ubo.get_input_image_buffer()+pad), 0, 1024, int(np.ceil(len(buff)/1024)))
+        self.input_texture_infos_ubo.get_input_image_buffer(self.input_texture_ubo_buffer.write)
        self.input_texture_infos_ubo_buffer.write(self.input_texture_infos_ubo.get_tex_data_buffer())
        self.user_input_ubo_buffer.write(self.user_input_ubo.to_bytes())
        out_data = self.user_output_ubo_buffer.read()
@@ -276,9 +356,10 @@ class MglApp(object):
        self.update_buffers()
        self.quad_fs.render(self.shader)

+import time

 class MglWindow(object):
-    def __init__(self, timer=None, args=None, backend="pygame2"):
+    def __init__(self, timer=None, args=["--vs=1"], backend="pygame2"):
        if backend is not None:
            available = mgw.find_window_classes()
            assert backend in available, f"backend {backend} is not installed. Installed backends: {available}"
@@ -343,6 +424,8 @@ class MglWindow(object):
    def imshow(self, window_name, frame):
        if frame.dtype == np.uint8:
            frame = frame.astype(np.float32) / 255
+        elif frame.dtype != np.float32:
+            frame = frame.astype(np.float32)

        if window_name in self.window_names.keys():
            i = self.window_names[window_name]
@@ -359,9 +442,10 @@ class MglWindow(object):
        # Always bind the window framebuffer before calling render
        self.window.use()

-        self.window.render(current_time, delta)
        if not self.window.is_closing:
+            self.window.render(current_time, delta)
            self.window.swap_buffers()
+            time.sleep(0)
        else:
            _, duration = self.timer.stop()
            self.window.destroy()
@@ -24,6 +24,7 @@ layout(std430, binding = 1) buffer TexData {
 };

 layout(std430, binding=2) buffer UserInput {
+    int sel_level;
    vec2 iMouse;
 };

@@ -47,26 +48,26 @@ void main() {
    float x_current = -1;
    vec2 coord;

-    for(int i=0;i<levels;i++){
-        if(bool(texLevels[i].flags&TEX_FLAG_HW)){
+    for (int i = 0;i < levels; i++) {
+        if (bool(texLevels[i].flags & TEX_FLAG_HW)) {
            coord = gl_FragCoord.yx;
-        }else{
+        } else {
            coord = gl_FragCoord.xy;
        }
-        if(coord.x>=texLevels[i].rect.x &&
-            coord.y>=texLevels[i].rect.y &&
-            coord.x<texLevels[i].rect.z &&
-            coord.y<texLevels[i].rect.w
-        ){
+        if (coord.x >= texLevels[i].rect.x &&
+        coord.y >= texLevels[i].rect.y &&
+        coord.x < texLevels[i].rect.z &&
+        coord.y < texLevels[i].rect.w
+        ) {
            our_level = i;
            //don't break. All shader instances should get same execution, and this puts later textures on top.
        }
    }

-    if(our_level!=-1) {
-        if(bool(texLevels[our_level].flags&TEX_FLAG_HW)){
+    if (our_level != -1) {
+        if (bool(texLevels[our_level].flags & TEX_FLAG_HW)) {
            coord = gl_FragCoord.yx;
-        }else{
+        } else {
            coord = gl_FragCoord.xy;
        }

@@ -82,7 +83,7 @@ void main() {
        int bottomRightIdx = topRightIdx + channels;

        //leave this for visual debugging
-        out_color = vec4(float(y_current)/float(levelHeight), float(x_current)/float(levelWidth), 0.0, 1.0);
+        out_color = vec4(float(y_current) / float(levelHeight), float(x_current) / float(levelWidth), 0.0, 1.0);

        out_color.x = bilinearInterpolation(
            fract(x_current),
@@ -96,33 +97,49 @@ void main() {
            out_color.y = bilinearInterpolation(
                fract(x_current),
                fract(y_current),
-                inputImage[bottomLeftIdx+1],
-                inputImage[bottomRightIdx+1],
-                inputImage[topLeftIdx+1],
-                inputImage[topRightIdx+1]
+                inputImage[bottomLeftIdx + 1],
+                inputImage[bottomRightIdx + 1],
+                inputImage[topLeftIdx + 1],
+                inputImage[topRightIdx + 1]
            );
        }
-        if(channels>2){
+        if (channels > 2) {
            out_color.z = bilinearInterpolation(
                fract(x_current),
                fract(y_current),
-                inputImage[bottomLeftIdx+2],
-                inputImage[bottomRightIdx+2],
-                inputImage[topLeftIdx+2],
-                inputImage[topRightIdx+2]
+                inputImage[bottomLeftIdx + 2],
+                inputImage[bottomRightIdx + 2],
+                inputImage[topLeftIdx + 2],
+                inputImage[topRightIdx + 2]
            );
        }
-        if(bool(texLevels[our_level].flags&TEX_FLAG_BGR)){
+        if (bool(texLevels[our_level].flags & TEX_FLAG_BGR)) {
            float temp_color = out_color.x;
            out_color.x = out_color.z;
            out_color.z = temp_color;
        }
        // currently only supporting 3 channels at most.
-    }else{
+    } else {
        // nice white background. ( ∩´ ᐜ `∩)
        out_color = vec4(1.0, 1.0, 1.0, 1.0);
    }

+    if (sel_level != -1) {
+        if (coord.x >= texLevels[sel_level].rect.x - 1 &&
+        coord.y >= texLevels[sel_level].rect.y - 1 &&
+        coord.x <= texLevels[sel_level].rect.z &&
+        coord.y <= texLevels[sel_level].rect.w
+        ) {
+            if (coord.x == texLevels[sel_level].rect.x - 1 ||
+            coord.y == texLevels[sel_level].rect.y - 1 ||
+            coord.x == texLevels[sel_level].rect.z ||
+            coord.y == texLevels[sel_level].rect.w
+            ) {
+                out_color = vec4(0.0, 0.5, 0.0, 1.0); // green selection border, on top of everything
+            }
+        }
+    }
+
    if(distance(iMouse, gl_FragCoord.xy)==0){
        hit_level = our_level;
        //hit_pos = vec2(x_current, y_current);
@@ -62,7 +62,7 @@ class SubscriberWindows(object):
        self.ctx = None
        self.sock_list: List[zmq.Socket] = []
        self.top_list: List[bytes] = []
-        self.displayer = mglwindow.MglWindow()
+        self.displayer = None

        if callbacks is None:
            callbacks = []
@@ -70,6 +70,7 @@ class SubscriberWindows(object):
            self.add_source(name)
        self.callbacks = callbacks
        if not self.silent:
+            self.displayer = mglwindow.MglWindow()
            for name in window_names:
                self.add_window(name)

@@ -77,10 +78,10 @@ class SubscriberWindows(object):

    def __bool__(self):
        self.update()
-        return not self.exited and not self.displayer.window.is_closing
+        return not self.exited and (self.displayer is None or not self.displayer.window.is_closing)

    def __iter__(self):
-        while not self.exited and not self.displayer.window.is_closing:
+        while not self.exited and (self.displayer is None or not self.displayer.window.is_closing):
            self.update()
            yield self.frames

@@ -295,7 +296,8 @@ class SubscriberWindows(object):
        self.__stop_all_cams()
        for t in self.close_threads:
            t.join()
-        self.displayer.window.close()
+        if self.displayer is not None:
+            self.displayer.window.close()

    def __enter__(self):
        return self
@@ -314,7 +316,7 @@ class SubscriberWindows(object):
        sub_cmd = window_commands.win_cmd_sub()
        msg_cmd = ""
        key = ""
-        while msg_cmd != "quit" and key != "quit" and (not self.displayer.window.is_closing):
+        while msg_cmd != "quit" and key != "quit" and (self.displayer is None or not self.displayer.window.is_closing):
            msg_cmd, key = self.update()
        sub_cmd.release()
        window_commands.quit(force_all_read=False)
@@ -1,14 +1,20 @@
 from displayarray import display
 import numpy as np

-arr = np.random.normal(0.5, 0.1, (100, 100, 3))
-arr2 = np.random.normal(0.5, 0.1, (200, 200, 3))
-arr3 = np.random.normal(0.5, 0.1, (300, 300, 3))
+arr = np.random.normal(0.5, 0.1, (100, 200, 3))
+arr2 = np.random.normal(0.5, 0.1, (200, 300, 3))
+arr3 = np.random.normal(0.5, 0.1, (300, 400, 3))

 with display(arr) as displayer:
    while displayer:
-        arr[:] += np.random.normal(0.001, 0.0005, (100, 100, 3))
+        arr[:] += np.random.normal(0.001, 0.0005, (100, 200, 3))
        arr %= 1.0
+        arr2[:] += np.random.normal(0.002, 0.0005, (200, 300, 3))
+        arr2 %= 1.0
+        arr3[:] -= np.random.normal(0.001, 0.0005, (300, 400, 3))
+        arr3 %= 1.0
+
+
        displayer.update(arr2, '2')
        displayer.update(arr3, '3')