If you reverse the borders it seems to work.

Code:
    Private Sub PictureBox_MouseEnter(sender As Object, e As EventArgs) _
        Handles PictureBox1.MouseEnter, PictureBox2.MouseEnter, PictureBox3.MouseEnter

        Dim pb As PictureBox = DirectCast(sender, PictureBox)
        'pb.BorderStyle = BorderStyle.FixedSingle
        pb.BorderStyle = BorderStyle.None '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
        ' Debug.WriteLine("E " & pb.Name)
    End Sub

    Private Sub PictureBox_MouseLeave(sender As Object, e As EventArgs) _
        Handles PictureBox1.MouseLeave, PictureBox2.MouseLeave, PictureBox3.MouseLeave

        Dim pb As PictureBox = DirectCast(sender, PictureBox)
        'pb.BorderStyle = BorderStyle.None
        pb.BorderStyle = BorderStyle.FixedSingle '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
        ' Debug.WriteLine("X " & pb.Name)
    End Sub