I may be oversimplifying this but why not just query for the combo before adding a new row?
declare @n1 int = 2
declare @n2 int = 1
declare @p int = 21
IF NOT EXISTS (SELECT 1 FROM MyTable
WHERE (col1 = @n1 AND col2 = @n2)
OR (col1 = @n2 AND col2 = @n1))
BEGIN
INSERT INTO MyTable VALUES (@n1,@n2,@p)
END
ELSE
BEGIN
UPDATE MyTable SET [priority]=@p
WHERE (col1 = @n1 AND col2 = @n2)
OR (col1 = @n2 AND col2 = @n1)
END