I know of no plugin that does this.

However, some custom coding displaying every letter regardless of having posts or not: http://wordpress.pastebin.com/f524c9f4

<?php
// Reference: http://wordpress.org/support/topic/148727

$posts = new WP_Query("cat=$cat&orderby=title&order=ASC&showposts=-1");
if ($posts->have_posts()) :
	for($i='A';$i!='AA';$i++) :
?>
<h3 id="<?php echo $i; ?>"></a><?php echo $i; ?></h3>
	<ul>
<?php
		while ($posts->have_posts()) :
			$posts->the_post();
			if( $i == strtoupper(trim(substr($post->post_title, 0, 1))) ) :
?>
	<li><a href="<?php the_permalink(); ?>" rel="bookmark" title="<?php the_title_attribute(); ?>"><?php the_title(); ?></a></li>
<?php
			endif;
		endwhile;
?>
	</ul>
<?php
	endfor;
endif;
?>

Version that only displays letter heading when one or more posts fall under it: http://wordpress.pastebin.com/f2302b7cb

<?php
// Reference: http://wordpress.org/support/topic/148727

$posts = new WP_Query("cat=$cat&orderby=title&order=ASC&showposts=-1");
if ($posts->have_posts()) :
	for($i='A';$i!='AA';$i++) :
?>
<?php
		while ($posts->have_posts()) :
			$posts->the_post();
			if( $i == strtoupper(trim(substr($post->post_title, 0, 1))) ) :
				if( !$a_z_header ) :
					$a_z_header = 1;
?>
<h3 id="<?php echo $i; ?>"></a><?php echo $i; ?></h3>
	<ul>
<?php
				endif;
?>
	<li><a href="<?php the_permalink(); ?>" rel="bookmark" title="<?php the_title_attribute(); ?>"><?php the_title(); ?></a></li>
<?php
			endif;
		endwhile;
?>
	</ul>
<?php
		$a_z_header = 0;
	endfor;
endif;
?>

Or try this:

<?php
   $args = array(
     'orderby' => 'title',
     'order' => 'ASC',
     'caller_get_posts' => 1,
     'posts_per_page' => 20,
                 );
query_posts($args);
if (have_posts()) {
  $curr_letter = '';
  while (have_posts()) {
    the_post();
    $this_letter = strtoupper(substr($post->post_title,0,1));
    if ($this_letter != $curr_letter) {
      echo "<h2> # $this_letter #</h2>";
      $curr_letter = $this_letter;
    }
?>
<br />
<a href="<?php the_permalink() ?>" rel="bookmark" title="Permanent Link to <?php the_title_attribute(); ?>"><?php the_title(); ?></a>
  <?php }
}
?>