I would also expect that using AcDbEntity::setVisibility() is the most performant way to change visibilities.
The show/hide performance should be comparable to thawing/freezing layers.
When you tried setVisibility(), did you force the GS (graphic system) somehow to update each entity after you changed its visibility? I.e. by calling AcDbEntity::draw() / AcDbEntity::recordGraphicsModified() / acedUpdateDisplay() / actrTransactionManager.flushGraphics() / acedRedraw() / ...? This would slow down the performance. It is better to do all changes within a transaction and let the GS do the update for all modified entities at once.
You might want to test performance with this simple code that hides/shows all modelspace entities on layer 0:
void visOnLayerId(AcDbObjectId idLayer, AcDb::Visibility vis)
{
AcDbDatabase *pDB = acdbHostApplicationServices()->workingDatabase();
AcDbObjectId IdModelSpace = acdbSymUtil()->blockModelSpaceId(pDB);
AcDbBlockTableRecord *model;
Acad::ErrorStatus es;
if ( (es=acdbOpenObject(model, IdModelSpace, AcDb::kForRead)) == Acad::eOk )
{
AcDbBlockTableRecordIterator *pit;
es = model->newIterator(pit);
if (pit)
{
for (pit->start(); !pit->done(); pit->step())
{
AcDbEntity *entity=nullptr;
es = pit->getEntity(entity, AcDb::kForRead);
if (!es)
{
if (entity->layerId() == idLayer)
{
if (entity->visibility()!=vis)
{
if (!entity->upgradeOpen())
entity->setVisibility(vis);
}
}
entity->close();
}
}
delete pit;
}
model->close();
}
}
void cmdHide()
{
AcDbDatabase *pDB = acdbHostApplicationServices()->workingDatabase();
AcDbObjectId idLayer0 = acdbSymUtil()->layerZeroId(pDB);
visOnLayerId(idLayer0, AcDb::kInvisible);
}
void cmdShow()
{
AcDbDatabase *pDB = acdbHostApplicationServices()->workingDatabase();
AcDbObjectId idLayer0 = acdbSymUtil()->layerZeroId(pDB);
visOnLayerId(idLayer0, AcDb::kVisible);
}
Thomas Brammer ● Software Developer ● imos AG ● LinkedIn ● 
If an answer solves your problem please [ACCEPT SOLUTION]. Otherwise explain why not.