Hi,
You have already specified the target control id as button then why you are again showing the modal pop up on button click.(its not required)
Have a look on the following code.
<asp:ScriptManager ID="ScriptManager1" runat="server">
</asp:ScriptManager>
<asp:Button ID="Button1" runat="server" Text="Button" />
<asp:ModalPopupExtender ID="Button1_ModalPopupExtender" runat="server" TargetControlID="Button1"
PopupControlID="Panel1" CancelControlID="btnCancel">
</asp:ModalPopupExtender>
<asp:Panel ID="Panel1" runat="server">
<asp:RadioButtonList ID="RadioButtonList1" runat="server">
<asp:ListItem Value="1">Test</asp:ListItem>
<asp:ListItem Value="2">Test 2</asp:ListItem>
<asp:ListItem Value="3">Test 3</asp:ListItem>
</asp:RadioButtonList>
<asp:Button ID="btnCancel" runat="server" Text="Cancel" />
</asp:Panel>
Hope this helps