I am trying to trigger frame capture at 30Hz using a 10% duty cycle with an exposure time of 10000 microseconds using a blackfly US 23S6M.
I am losing a significant amount of frames on two computers. One has a lot of RAM (>64GB) and the other has 16 GB. Both have hardrives that meet minimum specs. At 30 Hz for 8 seconds I should be ~240 frames. I am only getting ~168.
I am using pyspin capture and some threading and enabling maximum buffering. Is there any other way to fix this?
import PySpin
import time
import threading
import queue
import os
from datetime import datetime
import numpy as np
import skvideo
skvideo.setFFmpegPath('C:/Users/ksalvati/ffmpeg/bin') # set path to ffmpeg installation before importing io
import skvideo.io
import serial
import pickle
# constants
SAVE_FOLDER_ROOT = 'C:/Users/ksalvati/Documents/ISOI'
FILENAME_ROOT = 'KAS_' # optional identifier
EXPOSURE_TIME = 10000 # in microseconds
GAIN_VALUE = 25.0 # in dB, 0-40
IMAGE_HEIGHT = 1200 # 240 #540 pixels default
IMAGE_WIDTH = 1920 # 320 #720 pixels default
FRAME_RATE = 30 # Frame rate in Hz
BUFFER_SIZE = 840 # Set the desired buffer size here
SERIAL_PORT = 'COM6' # Change to your Arduino's serial port
SERIAL_BAUDRATE = 9600 # Baudrate for serial communication
# Expected frame interval in seconds (for 30 FPS)
EXPECTED_INTERVAL = 1.0 / FRAME_RATE
# generate output video directory and filename and make sure not overwriting
now = datetime.now()
mouseStr = input("Enter mouse ID: ")
dateStr = now.strftime("%Y_%m_%d") # save folder ex: 2020_01_01
timeStr = now.strftime("%H_%M_%S")
saveFolder = os.path.join(SAVE_FOLDER_ROOT, dateStr)
if not os.path.exists(saveFolder):
os.mkdir(saveFolder)
os.chdir(saveFolder)
movieName = FILENAME_ROOT + timeStr + '_' + mouseStr + '.mp4'
fullFilePath = os.path.join(saveFolder, movieName)
logFilePath = os.path.join(saveFolder, 'timestamps.pkl')
print('Video will be saved to: {}'.format(fullFilePath))
print('Timestamps will be saved to: {}'.format(logFilePath))
def initCam(cam, buffer_size): # function to initialize camera parameters for synchronized capture
cam.Init()
# load default configuration
cam.UserSetSelector.SetValue(PySpin.UserSetSelector_Default)
cam.UserSetLoad()
# set acquisition. Continuous acquisition. Auto exposure off. Set frame rate using exposure time.
cam.AcquisitionMode.SetValue(PySpin.AcquisitionMode_Continuous)
cam.ExposureAuto.SetValue(PySpin.ExposureAuto_Off)
cam.ExposureMode.SetValue(PySpin.ExposureMode_Timed) # Timed or TriggerWidth (must comment out trigger parameters other that Line)
cam.ExposureTime.SetValue(EXPOSURE_TIME)
# set analog. Set Gain + Gamma.
cam.GainAuto.SetValue(PySpin.GainAuto_Off)
cam.Gain.SetValue(GAIN_VALUE)
cam.PixelFormat.SetValue(PySpin.PixelFormat_Mono8)
cam.Width.SetValue(IMAGE_WIDTH)
cam.Height.SetValue(IMAGE_HEIGHT)
# setup FIFO buffer
s_nodemap = cam.GetTLStreamNodeMap()
node_buffer_handling_mode = PySpin.CEnumerationPtr(s_nodemap.GetNode('StreamBufferHandlingMode'))
if PySpin.IsAvailable(node_buffer_handling_mode) and PySpin.IsWritable(node_buffer_handling_mode):
node_oldest_first = node_buffer_handling_mode.GetEntryByName('OldestFirst')
if PySpin.IsAvailable(node_oldest_first) and PySpin.IsReadable(node_oldest_first):
buffer_handling_mode = node_oldest_first.GetValue()
node_buffer_handling_mode.SetIntValue(buffer_handling_mode)
print('Buffer handling mode set to OldestFirst...')
else:
print('Unable to set buffer handling mode to OldestFirst (entry retrieval). Aborting...')
return False
else:
print('Unable to set buffer handling mode (node retrieval). Aborting...')
return False
# Set the number of buffers used
node_buffer_count = PySpin.CIntegerPtr(s_nodemap.GetNode('StreamBufferCountManual'))
if PySpin.IsAvailable(node_buffer_count) and PySpin.IsWritable(node_buffer_count):
node_buffer_count.SetValue(buffer_size)
print(f'Buffer count set to {buffer_size}...')
else:
print('Unable to set buffer count. Aborting...')
return False
# set trigger input to Line0 (the black wire) if desired - default is Trigger OFF to free-run as fast as possible
cam.TriggerMode.SetValue(PySpin.TriggerMode_Off)
cam.TriggerSource.SetValue(PySpin.TriggerSource_Line0)
cam.TriggerActivation.SetValue(PySpin.TriggerActivation_RisingEdge) # LevelHigh or RisingEdge
cam.TriggerSelector.SetValue(PySpin.TriggerSelector_FrameStart) # require trigger for each frame
cam.TriggerMode.SetValue(PySpin.TriggerMode_On)
# optionally send exposure active signal on Line 2 (the white wire)
cam.LineSelector.SetValue(PySpin.LineSelector_Line1)
cam.LineMode.SetValue(PySpin.LineMode_Output)
cam.LineSource.SetValue(PySpin.LineSource_ExposureActive) # route desired output to Line 1 (try Counter0Active or ExposureActive)
def save_img(image_queue, writer, stop_event): # function to save video frames from the queue in a separate thread
while not stop_event.is_set() or not image_queue.empty():
try:
dequeuedImage = image_queue.get(timeout=1)
if dequeuedImage is not None:
writer.writeFrame(dequeuedImage)
image_queue.task_done()
except queue.Empty:
continue
def log_timestamps(ser, timestamp_queue, stop_event, arduino_start_time):
while not stop_event.is_set():
if ser.in_waiting > 0:
line = ser.readline().decode('utf-8').strip()
if line.isdigit():
arduino_time = int(line)
adjusted_time = arduino_time - arduino_start_time
timestamp_queue.put(adjusted_time)
print(f"Arduino: {arduino_time}, Adjusted: {adjusted_time}")
def main():
system = PySpin.System.GetInstance() # Get camera system
cam_list = system.GetCameras() # Get camera list
cam1 = cam_list[0]
initCam(cam1, BUFFER_SIZE)
# setup output video file parameters (can try H265 in future for better compression):
ffmpegThreads = 12 # this controls tradeoff between CPU usage and memory usage; video writes can take a long time if this value is low
writer = skvideo.io.FFmpegWriter(movieName, outputdict={'-vcodec': 'libx264', '-crf': str(crfOut), '-threads': str(ffmpegThreads)})
# Set up serial connection
ser = serial.Serial(SERIAL_PORT, SERIAL_BAUDRATE, timeout=1)
timestamp_queue = queue.Queue() # Queue to store timestamps
stop_event = threading.Event()
arduino_timestamps = []
frame_timestamps = []
dropped_frames = []
# Synchronize with Arduino start time
while ser.in_waiting == 0:
time.sleep(0.1)
arduino_start_time = int(ser.readline().decode('utf-8').strip())
python_start_time = time.time()
try:
print('Press Ctrl-C to exit early and save video')
cam1.BeginAcquisition()
tStart = time.time()
image_queue = queue.Queue() # create queue in memory to store images while asynchronously written to disk
# setup another thread to accelerate saving, and start immediately:
save_thread = threading.Thread(target=save_img, args=(image_queue, writer, stop_event))
save_thread.start()
# Setup timestamp logging thread
log_thread = threading.Thread(target=log_timestamps, args=(ser, timestamp_queue, stop_event, arduino_start_time))
log_thread.start()
frame_count = 0
last_frame_time = None # To track the last frame timestamp
while True:
image = cam1.GetNextImage() # get pointer to next image in camera buffer; blocks until image arrives via USB; timeout=INF
if image.IsIncomplete():
print(f'Image incomplete with image status {image.GetImageStatus()}')
image.Release()
continue
enqueuedImage = np.array(image.GetData(), dtype="uint8").reshape((image.GetHeight(), image.GetWidth())) # convert PySpin ImagePtr into numpy array
image_queue.put(enqueuedImage) # put next image in queue
capture_time = time.time()
frame_time = capture_time - python_start_time
print(f'Recorded frame # {frame_count} at {frame_time}')
frame_count += 1
# Log the frame capture time
frame_timestamps.append(frame_time)
# Detect dropped frames
if last_frame_time is not None:
interval = frame_time - last_frame_time
if interval > 1.5 * EXPECTED_INTERVAL:
dropped_frames.append((last_frame_time, frame_time))
print(f"Dropped frames detected between {last_frame_time} and {frame_time}")
last_frame_time = frame_time
# Log the Arduino timestamp
if not timestamp_queue.empty():
arduino_time = timestamp_queue.get()
arduino_timestamps.append(arduino_time)
image.Release() # release from camera buffer
except KeyboardInterrupt: # if user hits Ctrl-C, everything should end gracefully
stop_event.set()
cam1.EndAcquisition()
tEndAcq = time.time()
print('Capture ends at: {:.2f}sec'.format(tEndAcq - tStart))
# Calculate and print frame rate
elapsedTime = tEndAcq - tStart
image_queue.join() # wait until queue is done writing to disk
tEndWrite = time.time()
print('File written at: {:.2f}sec'.format(tEndWrite - tStart))
writer.close()
del image
cam1.DeInit()
del cam1
cam_list.Clear()
del cam_list
system.ReleaseInstance()
del system
# Save timestamps and dropped frames to pickle file
timestamps_data = {
'arduino_timestamps': np.array(arduino_timestamps),
'frame_timestamps': np.array(frame_timestamps),
'dropped_frames': dropped_frames
}
with open(logFilePath, 'wb') as pklfile:
pickle.dump(timestamps_data, pklfile)
print('Done!')
if __name__ == "__main__":
main()
Comments
8 comments