python – PyQT5 and OpenCv2 loading images incorrectly when refreshing

In a nutshell, I currently am trying to load a picture and view it using pyqt5, via QtWidgets.QGraphicsScene.

However, after loading the picture, I run some opencv2 commands (canny, gaussian to get some region-of-interest). These region-of-interest are simply XY coordinates. This then breaks the displaying picture — and I get lots of blue screens.

The picture and opencv2 operations are showcased below with the working example:

MAC OS, Python3.7 pyqt5 opencv

import sys
import os
from PyQt5 import QtCore, QtGui, QtWidgets

from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *

    
from PIL.ImageQt import ImageQt 
from PIL import Image
import cv2


# Convert an opencv image to QPixmap
def convertCvImage2QtImage(cv_img):
    rgb_image = cv2.cvtColor(cv_img, cv2.COLOR_BGR2RGB)
    PIL_image = Image.fromarray(rgb_image).convert('RGB')
    return QtGui.QPixmap.fromImage(ImageQt(PIL_image))


from PyQt5.QtWidgets import QApplication, QWidget, QPushButton, QLabel
from PyQt5.QtCore import Qt, QMimeData
from PyQt5.QtGui import QDrag, QPixmap
import numpy as np


class Ui_Form(object):
    def setupUi(self, Form):
        if not Form.objectName():
            Form.setObjectName(u"Form")
        Form.resize(900, 712)
        Form.setAcceptDrops(True)
        
        self.frame_graphics_view = QGraphicsView(Form)
        self.frame_graphics_view.setObjectName(u"frame_graphics_view")
        self.frame_graphics_view.setEnabled(True)
        self.frame_graphics_view.setGeometry(QRect(10, 10, 700, 300))
        sizePolicy = QSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.frame_graphics_view.sizePolicy().hasHeightForWidth())
        self.frame_graphics_view.setSizePolicy(sizePolicy)
        self.frame_graphics_view.setMouseTracking(True)
        self.frame_graphics_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        self.frame_graphics_view.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        self.frame_graphics_view.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored)
        self.frame_graphics_view.setAlignment(Qt.AlignLeading|Qt.AlignLeft|Qt.AlignTop)
        self.frame_graphics_view.setRubberBandSelectionMode(Qt.ContainsItemBoundingRect)
        
        self.refresh_frame_button = QPushButton(Form)
        self.refresh_frame_button.setObjectName(u"refresh_frame_button")
        self.refresh_frame_button.setGeometry(QRect(720, 10, 80, 26))
        
        
        self.show_roi_checkbox = QCheckBox(Form)
        self.show_roi_checkbox.setObjectName(u"show_roi_checkbox")
        self.show_roi_checkbox.setGeometry(QRect(720, 50, 85, 21))
        
        self.retranslateUi(Form)

        QMetaObject.connectSlotsByName(Form)
    # setupUi


    def retranslateUi(self, Form):
        Form.setWindowTitle(QCoreApplication.translate("Form", u"Form", None))
        self.refresh_frame_button.setText(QCoreApplication.translate("Form", u"refresh", None))
        self.show_roi_checkbox.setText(QCoreApplication.translate("Form", u"Show Roi", None))



class MyScene(QtWidgets.QGraphicsScene):
    def __init__(self, parent):
        super(MyScene, self).__init__()
        
        self.parent = parent
        
        self.red_color = QColor(255, 0, 0, 180)
        self.green_color = QColor(255, 255, 0, 180)
        self.blue_color = QColor(255, 0, 255, 180)
        
        self.greenBrush = QBrush(self.green_color)
        
        self.pen = QPen(self.red_color)
        
        
        self.current_ellipses = []

            
    def add_ellipses(self, pts):
        
        to_add = []
        for (x,y) in pts:
            ellipse = QGraphicsEllipseItem()
            ellipse.setPen(self.pen)
            ellipse.setBrush(self.greenBrush)
            
            ellipse.setRect(x, y, 10, 10)
            
            ellipse.setFlag(QGraphicsItem.ItemIsMovable)
            ellipse.setFlag(QGraphicsItem.ItemIsFocusable)
            ellipse.setFlag(QGraphicsItem.ItemIsSelectable)

            to_add.append(ellipse)
        
        for ellipse in to_add:
            self.addItem(ellipse)
            
        self.current_ellipses+=to_add
        
    def remove_current_ellipses(self):
        for item in self.current_ellipses:
            self.removeItem(item)
             
        self.current_ellipses = []
        
        

class ImageLoader(QtWidgets.QWidget):
    def __init__(self):
        QtWidgets.QWidget.__init__(self)
        
        self.ui = Ui_Form()
        self.ui.setupUi(self)
        
        
        self.ui.refresh_frame_button.clicked.connect(self.refresh_frame_only)
        self.ui.show_roi_checkbox.toggled.connect(self.show_roi)
        
        
        self.scene = QtWidgets.QGraphicsScene(self)
        self.scene = MyScene(self)
        self.ui.frame_graphics_view.setScene(self.scene)
        self.pixmap_item = self.scene.addPixmap(QtGui.QPixmap())
        
        # unclear why the UI is not setting this.. it should have.
        self.ui.frame_graphics_view.setDragMode(QGraphicsView.RubberBandDrag)
        
        
        self.go_to_image()
        
    def go_to_image(self):
        # get image
        img = self.get_image_to_show()
        if img is None:
            return
        
        # convert image
        pixmap = convertCvImage2QtImage(img)
        if pixmap.isNull():
            return
        
        # show image and determine if show ROI
        self.pixmap_item.setPixmap(pixmap)
        self.show_roi()
        
        
    def refresh_frame_only(self):
        # just refreshing, sometimes it fixes the underlying picture!
        self.go_to_image()
        
        
    def show_roi(self):
        '''Show region of interest in the picture.  This may require calculations.
        Lots of testing has shown that the more opencv2 commands that you call,
        the more likely it is to get the picture to "break".
        
        '''
        self.scene.remove_current_ellipses()
        if self.ui.show_roi_checkbox.isChecked():
            
            image = self.get_image_to_show()  # the image
            
            def calculate_roi():
                # run some opencv2 operations to find the ROI
                mask = cv2.Canny(image, 150, 250, 3)
                mask = cv2.bitwise_not(mask)
                
                image[mask!=0] = np.array((255,255,255))
                gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
                
                # Performing OTSU threshold
                ret, image_thresh = cv2.threshold(gray_image, 0, 255, cv2.THRESH_OTSU | cv2.THRESH_BINARY_INV)

                # we igonre douptuts
                ellipses = list([i, i] for i in [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 120, 130, 140, 150, 160, 170])
                return ellipses
            
            # as we decrease the range to say 1, it significantly decreases the chance of breaking the picture.
            for i in range(5):
                ellipses = calculate_roi()
                
            self.scene.add_ellipses(ellipses)
        else:
            self.scene.remove_current_ellipses()
        
        
    def get_image_to_show(self):
        '''LOAD A PICTURE'''
        image = cv2.imread('ducks.png')
        return image
        
#     

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    imageLoader = ImageLoader()
    imageLoader.show()
    sys.exit(app.exec_())

To reproduce the issue:

  1. Run the app.
  2. Click the checkbox so that it is Toggled
  3. Click Refresh many times. It can break on 1 click or 5-10 times.
  4. To “fix”, untoggle the checkbox and hit refresh.

Things I have tried:

  1. Remove the self.scene.add_ellipses — this removes the yellow dots, but still causes blue screen.
  2. decrease the opencv2 commands, this definitely helps.
  3. instead opencv2 commands do things like: for i in range(10000000): print. this runs slow…but no breaks

Very confused! any ideas are helpful

Example screenshots below:

  1. Loading pics. Family of ducks! Loaded yellow dots (note the toggle)
  2. Clicking refresh a few timestimes: Blue screens. Note partial loading of ducks!

enter image description hereenter image description here

Leave a Comment