a polyline . I have found a solution , that is transform each point in the polyline and create a new polyline. but what cnfuse me is that direrent order of transform cause different result.
Entity subEnt = trans.GetObject(id, OpenMode.ForRead) as Entity;
if (subEnt is Polyline)
{
Polyline tempPoly = subEnt as Polyline;
double rot = br.Rotation;
double xscale = br.ScaleFactors.X;
double yscale = br.ScaleFactors.Y;
double zscale = br.ScaleFactors.Z;
//// method 1 scale, rotate and move
double[] dd = new double[16];
dd[0] = xscale;
dd[5] = yscale;
dd[10] = zscale;
dd[15] = 1;
Matrix3d scaleMatrix = new Matrix3d(dd);
Polyline tp = new Polyline();
for (int i = 0; i < tempPoly.NumberOfVertices; i++)
{
Point3d p3 = tempPoly.GetPoint3dAt(i).TransformBy(scaleMatrix);
Point2d p2 = new Point2d(p3.X, p3.Y);
tp.AddVertexAt(i, p2, tempPoly.GetBulgeAt(i), tempPoly.GetStartWidthAt(i), tempPoly.GetEndWidthAt(i));
}
tempPoly = tp;
Matrix3d m3dRot = Matrix3d.Rotation(rot, new Vector3d(0, 0, 1), Point3d.Origin);
tempPoly.TransformBy(m3dRot);
Matrix3d m3dDisp = Matrix3d.Displacement(insertPnt.GetAsVector());
tempPoly.TransformBy(m3dDisp);
//// method 2: move, rotate and scale
//Matrix3d m3dDisp = Matrix3d.Displacement(insertPnt.GetAsVector());
//tempPolyline.TransformBy(m3dDisp);
//Matrix3d m3dRot = Matrix3d.Rotation(rot, new Vector3d(0, 0, 1), insertPnt);
//tempPolyline.TransformBy(m3dRot);
//double[] dd = new double[16];
//dd[0] = xscale;
//dd[3] = insertPnt.X * (1 - xscale);
//dd[5] = yscale;
//dd[7] = insertPnt.Y * (1 - yscale);
//dd[10] = zscale;
//dd[15] = 1;
//Matrix3d scaleMatrix = new Matrix3d(dd);
//Polyline tp = new Polyline();
//for (int i = 0; i < tempPolyline.NumberOfVertices; i++)
//{
// Point3d p3 = tempPolyline.GetPoint3dAt(i).TransformBy(scaleMatrix);
// Point2d p2 = new Point2d(p3.X, p3.Y);
// tp.AddVertexAt(i, p2, tempPolyline.GetBulgeAt(i), tempPolyline.GetStartWidthAt(i), tempPolyline.GetEndWidthAt(i));
//}
//tempPolyline = tp;
}