Try this,
Javascript:
<script type="text/javascript" language="javascript">
function isNumberKey(evt) {
var charCode = (evt.which) ? evt.which : event.keyCode
if (charCode > 31 && (charCode < 48 || charCode > 57))
return false;
return true;
}
<asp:CustomValidator CSSClass="body" id="cusPhone" ValidationGroup="EditProfile" onkeypress="return isNumberKey(event)" Text="Invalid mobile Number" runat="server" />