ロック用の追加のサブクエリレイヤーはどうですか?
UPDATE my_table
SET processing_by = our_id_info -- unique to this instance
WHERE trans_nbr IN (
SELECT trans_nbr
FROM my_table
WHERE trans_nbr IN (
SELECT trans_nbr
FROM my_table
GROUP BY trans_nbr
HAVING COUNT(trans_nbr) > 1
LIMIT our_limit_to_have_single_process_grab
)
FOR UPDATE
)
RETURNING row_id