well if you want to store binary data, you need to change the type of your column.
your md5 signature should also be changed.
you are missing your schema, so you would have to define your schema before loading your data.
So I fixed all that and rewrote your function for you, including generating the hash.
private string FillFileData(string file)
{
using (var xmlFile = new DataSet("File"))
{
using (var table = xmlFile.Tables.Add("Data"))
{
table.Columns.Add("Name", typeof(string));
table.Columns.Add("MD5", typeof(byte[]));
table.Columns.Add("data", typeof(byte[]));
byte[] b = File.ReadAllBytes(file);
var hasher = System.Security.Cryptography.MD5.Create();
var outputHash = hasher.ComputeHash(b);
var fileName = Path.GetFileName(file);
var oo = new object[] { fileName, outputHash, b };
table.Rows.Add(oo);
string outputFileName = Path.GetTempFileName();
using (var sw = new StreamWriter(outputFileName))
{
xmlFile.WriteXmlSchema(sw);
xmlFile.WriteXml(sw);
}
return outputFileName;
}
}
}