Edit in JSFiddle

Result
JavaScript
HTML
CSS

var map, markers;


map = new OpenLayers.Map("map");
map.addLayer(new OpenLayers.Layer.OSM());

var virginia = new OpenLayers.LonLat(-78.65689420, 37.43157340).transform(
  new OpenLayers.Projection("EPSG:4326"),
  map.getProjectionObject()
);

var cville = new OpenLayers.LonLat(-78.47667810000002, 38.02930590).transform(
  new OpenLayers.Projection("EPSG:4326"),
  map.getProjectionObject()
);

var alderman = new OpenLayers.LonLat(-78.50572199999999, 38.03467470).transform(
  new OpenLayers.Projection("EPSG:4326"),
  map.getProjectionObject()
);

var size = new OpenLayers.Size(21,25);
var offset = new OpenLayers.Pixel(-(size.w/2), -size.h);
var blue = new OpenLayers.Icon('http://www.openlayers.org/dev/img/marker-blue.png', size, offset);
var green = new OpenLayers.Icon('http://www.openlayers.org/dev/img/marker-green.png', size, offset);


var zoom=7;

var markers = new OpenLayers.Layer.Markers( "Markers" );
map.addLayer(markers);

markers.addMarker(new OpenLayers.Marker(virginia));
markers.addMarker(new OpenLayers.Marker(cville, blue));
//markers.addMarker(new OpenLayers.Marker(alderman, green));

map.setCenter (cville, zoom);

<!DOCTYPE HTML>
<html>
    <head>
        <meta http-equiv="content-type" content="text/html; charset=utf-8" />
        <title>Example Map</title>
        <script src="http://openlayers.org/api/OpenLayers.js"></script>
    </head>
    <body>
        <div id="map"></div>
    </body>
</html>

#map {width: 800px; height: 600px;}