The solution was very simple - I just added RelativeSource to my binding in Mode = FindAncestor.
Code looks now so:
<window.resources>
<resourcedictionary>
<style targettype="{x:Type propgrid:PropertyGridDataAccessorItem}">
<style.triggers>
<trigger property="DataAccessorType" value="Category">
<setter property="IsExpanded">
<setter.value>
<multibinding>
<multibinding.converter>
<local:expandedcategoryconverter />
</multibinding.converter>
<binding path="DisplayName" relativesource="{RelativeSource Self}" />
<binding path="ViewModel" relativesource="{RelativeSource Mode=FindAncestor, AncestorType=Window}" />
</multibinding>
</setter.value>
</setter>
</trigger>
</style.triggers>
</style>
</resourcedictionary>
</window.resources>