You need to lock the image, and then work with memory directly, bypassing the SetPixel method.
See here or even better here
For reference, you can set the blue color to 255 as follows
BitmapData bmd=bm.LockBits(new Rectangle(0, 0, 10, 10), System.Drawing.Imaging.ImageLockMode.ReadOnly, bm.PixelFormat);
int PixelSize=4;
for(int y=0; y<bmd.Height; y++)
{
byte* row=(byte *)bmd.Scan0+(y*bmd.Stride);
for(int x=0; x<bmd.Width; x++)
{
row[x*PixelSize]=255;
}
}