You have to use onclientclick of button and style="visibility:hidden;" property of panel,make it visible from javascript.Try following code
<asp:Panel runat="server" ID="pnlHeader" Visible="true">
Search:<br />
<asp:TextBox runat="server" id="tbxSearch" />
<asp:RequiredFieldValidator runat="server" id="reqSearch" controltovalidate="tbxSearch" errormessage="(Required)" />
<asp:Button runat="server" id="btnSearch" text="Search" OnClientClick="validateSearch()" />
</asp:Panel>
<br />
<br />
<asp:Panel ID="pnlUpload" runat="server" style="visibility:hidden;">
<asp:FileUpload ID="fileUpload" runat="server" />
<asp:Button ID="btnUpload" Text="Upload" runat="server" OnClick="btnUpload_Click" OnClientClick="validateUpload()" />
<asp:RequiredFieldValidator ID="reqFileUpload" runat="server" ControlToValidate="fileUpload" ErrorMessage="File Required!" />
</asp:Panel>
<script type="text/javascript">
function validateSearch()
{
var searchTxt = document.getElementById('tbxSearch').value ;
if(searchTxt != '')
{
document.getElementById('pnlUpload').style.visibility="visible";
return true;
}
else
return false;
}
function validateUpload()
{
if(document.getElementById("fileUpload").value != "" && document.getElementById('tbxSearch').value != "")
{
return true;
}
else
return false;
}
</script>