The problem was that you need to walk up the tree and select productgroup, then go down the tree again to find all lines.
Modified XSD
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0">
<xsl:output encoding="iso-8859-1" />
<xsl:output method="xml" indent="yes" omit-xml-declaration="yes" />
<xsl:key name="level0" match="//product" use="@ordercat" />
<xsl:template match="/">
<data>
<xsl:attribute name="type">K8 order creation control file</xsl:attribute>
<process><xsl:value-of select="count(/order/productgroup/supplier/product[count(. | key('level0', @ordercat)[1]) = 1])" /></process>
<xsl:for-each select="/order/productgroup/supplier/product[count(. | key('level0', @ordercat)[1]) = 1]">
<order>
<xsl:attribute name="number"><xsl:number format="1" value="position()" /></xsl:attribute>
<xsl:variable name="onrOrdCat" select="@ordercat" />
<ordercategory><xsl:value-of select="$onrOrdCat" /></ordercategory>
<orderlines>
<xsl:apply-templates select="../../../productgroup">
<xsl:with-param name="onrOrdCat" select="$onrOrdCat" />
</xsl:apply-templates>
</orderlines>
</order>
</xsl:for-each>
</data>
</xsl:template>
<xsl:template match="productgroup">
<xsl:param name="onrOrdCat" />
<xsl:apply-templates select="supplier">
<xsl:with-param name="onrOrdCat" select="$onrOrdCat" />
</xsl:apply-templates>
</xsl:template>
<xsl:template match="supplier">
<xsl:param name="onrOrdCat" />
<xsl:apply-templates select="product[@ordercat = $onrOrdCat]" />
</xsl:template>
<xsl:template match="product">
<line>
<xsl:attribute name="what"><xsl:value-of select="@what" /></xsl:attribute>
<xsl:attribute name="pos"><xsl:value-of select="@pos" /></xsl:attribute>
</line>
</xsl:template>
</xsl:stylesheet>