What you need is to generate a random permutation of 0-24, assigning one number for each of the buttons. Then just use number % 5 to assign the number and number / 5 to index into the color list.
To efficiently generate the random permutation try this:
using System.Collections.Generic;
using System.Linq;
private Random Rand = new Random();
private List<int> RandPerm(int N)
{
List<int> p = new List<int>(Enumerable.Range(0, N));
for (int k = 2; k <= N; k++)
{
int r = (int)Math.Ceiling(k * Rand.NextDouble());
--r;
p[k - 1] = p[r];
p[r] = k - 1;
}
return p;
}