It is not obvious to me that I was handling this completely wrong. Here is what I ended up with. FYI, I try to keep my screen as uncluttered as possible, hence the long modemacro string.
Double adfltscale = 0.375;
Database db = HostApplicationServices.WorkingDatabase;
Transaction trans;
trans = db.TransactionManager.StartTransaction();
double varltscale;
varltscale = (factor * adfltscale);
acadapp.SetSystemVariable("ltscale", varltscale);
acadapp.SetSystemVariable("Dimscale", factor);
acadapp.SetSystemVariable("Celtscale", 1);
acadapp.SetSystemVariable("Psltscale", 0);
acadapp.SetSystemVariable("Insunits", 1);
acadapp.SetSystemVariable("Dimasz", 0.125);
acadapp.SetSystemVariable("Lunits", 4);
acadapp.SetSystemVariable("Luprec", 6);
acadapp.SetSystemVariable("ModeMacro", "LTSCALE:$(GETVAR, LTSCALE) DIMSCALE:$(GETVAR, DIMSCALE) ARROW SIZE:$(GETVAR, DIMASZ) SCALE:$(rtos, $(/, 12, $(/, $(GETVAR, LTSCALE), .375))) TEXTSIZE:$(GETVAR, TEXTSIZE) DIMSTYLE:$(GETVAR, DIMSTYLE) LAYER: $(GETVAR, clayer) |");
trans.Commit();
trans.Dispose();