mattbrehmer · August 29, 2015 14:12
diff --git a/Readme.md b/Readme.md
diff --git a/albumranks.csv b/albumranks.csv
diff --git a/albumscores.csv b/albumscores.csv
diff --git a/aoty-combine.R b/aoty-combine.R
 require(data.table)

 albumscores.dt = scores.dt[albums.dt,allow.cartesian=T] #combine the data tables
 albumscores.dt = unique(albumscores.dt) #remove duplicate rows

 #remove duplicate columns
 albumscores.dt[,Artist.1:=NULL]
 albumscores.dt[,Album.1:=NULL]

 #coerce classes on columns, remove commas from character columns
 albumscores.dt$ReleaseDate = as.Date(albumscores.dt$ReleaseDate,format = "%B %d, %Y")
 albumscores.dt$url = as.factor(albumscores.dt$url)
 setnames(albumscores.dt,"url","Artist_url")
 albumscores.dt$Label = gsub(",","",albumscores.dt$Label)
 albumscores.dt$Label = as.factor(albumscores.dt$Label)
 albumscores.dt$Genre = gsub(",","",albumscores.dt$Genre)
 albumscores.dt$Genre = as.factor(albumscores.dt$Genre)
 albumscores.dt$Artist = gsub(",","",albumscores.dt$Artist)
 albumscores.dt$Artist = as.factor(albumscores.dt$Artist)
 albumscores.dt$Album = gsub(",","",albumscores.dt$Album)
 albumscores.dt$Album = as.factor(albumscores.dt$Album)

 #sort by artist, album
 setkey(albumscores.dt,Artist,Album)

 # albumscores.dt = albumscores.dt[order(-rank(AoTY))]
 # 
 # albumscores.dt = albumscores.dt[1:100]

 albumscores.dt = transform(albumscores.dt, SD=apply(albumscores.dt[,3:21, with = F],1, sd, na.rm = TRUE))

 #duplicate table for storing ranks
 albumranks.dt = albumscores.dt

 #ranking function applied to a numeric column
 convertToRank = function(col) {
  
  if(is.numeric(col)){
    col = rank(-col,na.last = "keep",ties.method = "min")
    col = findInterval(col, sort(unique(col)))
  }
  
  return(col)
 }
 #rank within numeric columns and replace with ranks
 albumranks.dt = albumranks.dt[, lapply(.SD, convertToRank)]

 #sort ranked table by artist, album
 setkey(albumranks.dt,Artist,Album)

 #copy rank column from score table to rank table
 albumranks.dt$rank = albumscores.dt$rank

 #remove rank column from score table
 albumscores.dt[,rank:=NULL]

 #remove AoTY rank from rank table (rank = AoTY rank)
 albumranks.dt[,AoTY:=NULL]

 #write to csv
 write.csv(albumranks.dt,file="albumranks.csv",quote=F,row.names=F,na="")
 write.csv(albumscores.dt,file="albumscores.csv",quote=F,row.names=F,na="")
 # write.csv(list_urls.dt,file="list_urls.csv",quote=F,row.names=F,na="")
diff --git a/aoty-preprocess.R b/aoty-preprocess.R
 #main pre-processing script

 #change year as necessary
 yr = 2014

 source("aoty-table-parse.R")
 source("aoty-scrape.R")
 source("aoty-combine.R")

 # require(data.table)

 #preprocess from previously cleaned csv files

 # list_urls.dt = as.data.table(read.csv(file="aoty-list-urls.csv",header = T))
 # setkey(list_urls.dt,name)
 # 
 # ranks.dt = as.data.table(read.csv(file="aoty-w-links.csv",header = T))
 # setkey(ranks.dt,Album_url)
diff --git a/aoty-scrape.R b/aoty-scrape.R
 require(data.table)

 albums.dt = data.table()

 scrapeAlbum = function(url) {
  
  print(c("scraping: ",url))
  
  raw.data = readLines(as.character(url),warn=F)
  
  #locate relevant metadata on page
  artistLoc = sapply('.*span class="item', function(y) grep(y,raw.data)[1])
  albumLoc = sapply('.*span class="fn', function(y) grep(y,raw.data)[1])
  releasedLoc = sapply('RELEASED:', function(y) grep(y,raw.data)[1])
  labelLoc = sapply('LABEL:', function(y) grep(y,raw.data)[1])
  genreLoc = sapply('GENRE:', function(y) grep(y,raw.data)[1])
  websiteLoc = sapply('WEBSITE:', function(y) grep(y,raw.data)[1])
  
  #clean the html off of the metadata
  artist = gsub(".*>(.*)</a> - <span class=\"fn\">.*","\\1",raw.data[artistLoc])
  artist = gsub(" *$","",artist) #remove trailing white space
  album = gsub(".*<span class=\"fn\">(.*)</span></span>.*","\\1",raw.data[albumLoc])
  album = gsub(" *$","",album) #remove trailing white space
  releaseDate = gsub(".*RELEASED: *</span> *(.*)</div>","\\1",raw.data[releasedLoc])
  releaseDate = gsub(" *$","",releaseDate) #remove trailing white space
  label = gsub(".*LABEL: *</span>(.*)</div>","\\1",raw.data[labelLoc])
  label = gsub(" *$","",label) #remove trailing white space
  
  website = gsub(".*WEBSITE:.*>(.*) *<img.*</a></div><div style.*","\\1",raw.data[websiteLoc])
  website = gsub(" *$","",website) #remove trailing white space
  
  #genre scraping depends on presence / absence of website
  if (is.na(website)) {
    genre = gsub(".*GENRE.*>(.*)</a>.*TAGS*","\\1",raw.data[genreLoc])
    genre = gsub("([A-Z]*,* *[A-Z]*)<.*","\\1",genre)
  }
  if (!is.na(website)) {
    genre = gsub(".*GENRE.*>(.*)</a>.*WEBSITE.*","\\1",raw.data[genreLoc])
    website = gsub("^","http://",website)
  }
  
  genre = gsub(" *$","",genre) #remove trailing white space
  
  remove(artistLoc,albumLoc,releasedLoc,labelLoc,genreLoc,websiteLoc,raw.data)
  
  return (data.table(artist,album,releaseDate,label,genre,website,url))
  
 }

 #scrape album metadata from scores table
 albums.dt = do.call("rbind", lapply(X=scores.dt$Album_url,FUN = scrapeAlbum))
 setnames(albums.dt, c("Artist","Album","ReleaseDate","Label","Genre","url","Album_url"))
 setkey(albums.dt,Album_url)
diff --git a/aoty-table-parse.R b/aoty-table-parse.R
 require(data.table)

 #scrape the aggregate score table for the specified year
 scrapeTable = function(year){
  
  url = paste("http://www.albumoftheyear.org/ratings/overall/",year,"/15",sep = "")
  
  raw.data = readLines(url,warn=F)
  
  #trim everything but the table
  tableBegin = sapply('<TABLE', function(y) grep(y,raw.data)[1])
  tableEnd = sapply('</TABLE>', function(y) grep(y,raw.data)[1])
  
  table = raw.data[tableBegin:tableEnd]
  
  return(table)
  
 }

 #change X in scrapeTable(X) to specify the year e.g. 2014
 table = scrapeTable(2014)

 table = unique(table) #remove duplicate header rows
 table = table[-26] #remove first duplicate header row

 #remove first list item from header
 header = gsub(".*(<tr class=\"white-font\".*)</a></td></tr><tr.*","\\1",table[1])

 #trim html from header
 header = gsub("</td>",",",header)
 header = gsub("<td>","",header)
 header = gsub("<tr class=\"white-font\" align=\"center\" bgcolor=\"#58808b\">","",header)
 header = gsub("<a class=\"white\" href=\"","",header,)
 header = gsub("</a>","",header,)
 header = gsub("\">",",",header,)

 header = strsplit(header,",") #split string into list
 colNames = c("rank","AoTY")
 reviewNames = header[[1]][((1:length(header[[1]]))%%2==0)]
 reviewNames = reviewNames[3:length(reviewNames)]
 aotyUrls = header[[1]][((1:length(header[[1]]))%%2==1)]
 aotyUrls = aotyUrls[3:length(aotyUrls)]
 colNames = c(colNames,reviewNames,"Artist","Profile_url","Album","Album_url")

 #remove header from row 1
 table[1] = gsub(".*(<tr bgcolor=\"#......\">.*)","\\1",table[1])

 #coerce chacacters to numeric
 numericFromChar = function(chr) {
  
  return (as.numeric(as.character(chr)))
  
 }

 #parse a single row of the table
 parseRow = function(str) {
  
  cols = t(as.data.table(strsplit(str,"<td")))
  cols = gsub(".*n/a.*",NA,cols)
  cols = gsub(".*>([0-9]+\\.*[0-9]*)<.*","\\1",cols)
  cols = cols[2:length(cols)]
  meta = t(as.data.table(strsplit(cols[3],"<div")))
  cols = as.data.table(t(as.data.table(cols[-3])))
  cols = cols[, lapply(.SD, numericFromChar)]
  artist = gsub(".*>(.*)</a.*","\\1",meta[2],)
  artist_url = gsub(".*a href=\"(.*/)\">.*","http://albumoftheyear.org\\1",meta[2],)
  album = gsub(".*>(.*)</a.*","\\1",meta[3],)
  album_url = gsub(".*a href=\"(.*php)\">.*","http://albumoftheyear.org\\1",meta[3],)
  
  meta = t(as.data.table(c(artist,artist_url,album,album_url)))
  
  row = as.data.table(cbind(cols,meta))
  
  setnames(row,colNames)
  
  return(row)
  
 }

 #parse the table
 scores.dt = do.call("rbind", lapply(table[1:length(table)-1], FUN = parseRow))
 setkey(scores.dt,Album_url)

 #remove rows with missing data
 scores.dt = unique(scores.dt[!"NA"])

 #clean review urls
 list_urls.dt = as.data.table(cbind(reviewNames,aotyUrls))
 list_urls.dt$aotyUrls = gsub("/ratings/","http://albumoftheyear.org/ratings/",list_urls.dt$aotyUrls)
 setkey(list_urls.dt,reviewNames)

 remove(colNames,reviewNames,table,header,aotyUrls)
diff --git a/index.html b/index.html
 <link rel = "stylesheet" type="text/css" href="style.css" />
 <body>
 	<script src="http://d3js.org/d3.v3.min.js"></script>
 	<script type="text/javascript" src="soundConsensus.js"></script>
 </body>
diff --git a/info-hover.png b/info-hover.png
diff --git a/info.png b/info.png
diff --git a/list_urls.csv b/list_urls.csv
diff --git a/soundConsensus.js b/soundConsensus.js
 /**

 SoundConsensus

 A visual summary of album review scores
 Data scraped from albumoftheyear.org

 by Matt Brehmer / @mattbrehmer

 December 2014

 **/

 //track horizontal and vertical scrolling
 window.pos = function() {
  if (window.scrollX != null && window.scrollY != null) 
    return { x: window.scrollX, y : window.scrollY };
  else 
    return { x: document.body.parentNode.scrollLeft, 
             y: document.body.parentNode.scrollTop };
 };

 //keep header panel fixed to top but not left
 window.onscroll = function(e){
  document.getElementById('header_panel')
          .style.top = window.pos().y + 'px';
 };

 //initialize dimensions
 var margin = {top: 20, right: 20, bottom: 20, left: 20},
    width = window.innerWidth - 35,
    height = 2440;

 //initialize scales
 var x = d3.scale.ordinal(),
    y = d3.scale.linear(), //scales for each dimension
    z = d3.scale.linear(), //scale for bar charts
    genre_scale = d3.scale.ordinal(), //ordinal scale for genres
    label_scale = d3.scale.ordinal(); //ordinal scale for labels

 //initialize dispatch for highlighting selections from dropdowns
 var dispatch = d3.dispatch("highlight");
                  

 //initialize main svg area
 var main_svg = d3.select("body")
                 .append("svg")
                 .attr("id", "main_panel")
                 .attr("width", width)
                 .attr("height", height);

 //initialize header svg
 var header_svg = d3.select("body")
                   .append("svg")
                   .attr("id", "header_panel")
                   .attr("width", width)
                   .attr("height", 25); 

 //initialize filter div
 var filter_div = d3.select("body")
                   .append("div") 
                   .attr("id", "filter_div");                     

 //initialize footer svg
 var footer_svg = d3.select("body")
                   .append("svg")
                   .attr("id", "footer_panel")
                   .attr("width", width)
                   .attr("height", 25) ;    

 //initialize tooltip svg
 var tooltip_svg = d3.select("body")
                    .append("svg")
                    .attr("id", "tooltip_panel")
                    .attr("class", "tooltip")
                    .attr("width", 160)
                    .attr("height", 120);                 

 //tooltip text and link fields
 tooltip_svg.append("a")
           .attr("id","tooltip_artist_link") 
           .append("text")
           .attr("dy", "0.9em")
           .attr("dx", "0.3em")
           .attr("class","artist")
           .attr("id","tooltip_artist");                                                

 tooltip_svg.append("a")
           .attr("id","tooltip_album_link")
           .append("text")
           .attr("dy", "1.9em")
           .attr("dx", "0.3em")
           .attr("class","album")
           .attr("id","tooltip_album");                                                

 tooltip_svg.append("text")
          .attr("dy", "3.9em")
          .attr("dx", "0.3em")
          .attr("id","tooltip_genre");                                                                                              

 tooltip_svg.append("text")
          .attr("dy", "4.9em")
          .attr("dx", "0.3em")
          .attr("id","tooltip_release");     

 tooltip_svg.append("text")
          .attr("dy", "6.9em")
          .attr("dx", "0.3em")
          .attr("id","tooltip_score");                                            

 tooltip_svg.append("a")
           .attr("id","tooltip_website_link")
           .append("text")
           .attr("dy", "8.9em")
           .attr("dx", "0.3em")
           .attr("class","album")
           .attr("id","tooltip_website");  

 var about_visible = false;              

 var about_panel = d3.select("body")
                    .append("div") 
                    .attr("id", "about_panel")    
                    .style("display",'none')
                    .on("click", function() {
                      about_visible = false;
                      d3.select(this).style("display","none");
                    })                                                  
                    .html('<strong>SoundConsensus</strong> is an interactive visualization by @<a href="https://twitter.com/mattbrehmer">mattbrehmer</a> for comparing multiple ranked lists of record reviews from 19 prominent music publications. ' + 
                      'The data visualized here represents the 105 most-reviewed records released in 2014, according to the music publication aggregator site <a href="http://www.albumoftheyear.org/ratings/overall/2014/15">albumoftheyear.org</a>. ' + 
                      '<br/><br/><strong>Visual Encoding</strong>: Each column is associated with a music publication. Each cell containing a bar corresponds to a review score. The vertical position of a cell encodes its rank among other reviews from that publication. ' + 
                      'The bars in each cell encode the score itself.<br/><br/>The first column is unique in that it encodes the overall rank and score calculated by <a href="http://www.albumoftheyear.org/ratings/overall/2014/15">albumoftheyear.org</a>. ' + 
                      '<br/><br/>The columns are of unequal size because: (1) not all of the music publications reviewed all of the records; and (2) some music publications use a 10-point scale when rating a record, resulting in more ties than those using a 100-point or decimal scale. ' + 
                      '<br/><br/><strong>Interaction</strong>: Hover over a record\'s artist or name to highlight the ranks and scores across all of the music publications who reviewed the record, and to see details about the record (such as genre, record label, and release date) in the panel at the lower left. ' + 
                      '<br/><br/>You can also hover over any cell. ' + 
                      'Clicking on a cell makes the highlighting persist, which can facilitate comparisons between records. Clicking again removes the highlight. ' +
                      '<br/><br/>Click on an artist name or album name to visit corresponding <a href="http://www.albumoftheyear.org">albumoftheyear.org</a> artist and album profile pages, which contain links to the original reviews. ' + 
                      '<br/><br/>Hover over a column header to see the corresponding music publication\'s full name in a tooltip, along with details about the publication in the panel at the lower left. ' +
                      '<br/><br/><strong>Genre / Record Label Filtering</strong>: Select a musical genre and / or record label from the dropdown boxes in the lower left to filter the list of records (filtering maintains the relative rank positions of review scores). ' +
                      '<br/><br/><strong>Consensus Filtering</strong>: Select a consensus level from the dropdown box in the lower left to filter based on a record\'s standard deviation of review scores, where a high standard deviation corresponds to a low consensus, and vice versa (consensus ranges are at 20% quantiles). ' +
                      '<br/><br/>(Click anywhere in this panel to close it.)');
              
 //create an array of known metadata dimensions              
 var metadata = ["Album_url", 
                "Artist",
                "Profile_url",
                "Album",
                "ReleaseDate",
                "Label",
                "Genre",
                "Artist_url",
                "AoTY",
                "SD"]

 //load the data from csv
 d3.csv("data-aoty/albumscores.csv", function(error, data) { 
  d3.csv("data-aoty/albumranks.csv", function(error, rank_data) { 
    d3.csv("data-aoty/list_urls.csv", function(error, reviewer_data) { 

      //sort SD array for consensus filtering
      var sd_arr = data.map(function(e) { return +e.SD; }).sort(d3.ascending);

      //get list of dimensions from rank data
      rank_dimensions = d3.keys(rank_data[0]);

      //sort the data based on albumoftheyear (AoTY) aggregate score
      data.sort(function(a,b) {
        return b.AoTY-a.AoTY;
      });

      // Extract the list of dimensions and create a scale for each.
      // dimensions are scores from review publications
      x.domain(dimensions = d3.keys(data[0]).filter(function(d) {
        return metadata.indexOf(d) == -1;
      }));

      //determine cell size based on the number of dimensions
      var cell_width = width / (dimensions.length + 4.5 );
      var cell_height = 16;

      x.rangePoints([cell_width * 4, 
        cell_width * 4 + dimensions.length * cell_width], 1);  

      //specify range and domain of bar charts based on cell width
      y.range([0,height - 100])
       .domain([0,data.length - 1]);

      //specify range and domain of bar charts based on cell width
      z.range([0,cell_width / 1.5])
       .domain([0,100]);

      //specify genre scale domain
      genre_scale.domain(data.map( function (d) { 
        return d.Genre; 
      }));

      //specify label scale domain
      label_scale.domain(data.map( function (d) { 
        return d.Label; 
      }));

      /**

      HEADER

      **/

      //append container containing column heads to header panel
      var header = header_svg.append("g")
                             .attr("class","header");

      //append title to header                       
      header.append("text")
            .attr("class","title")
            .attr("dy", "0.7em")
            .text(function() {
              if (width >= 1400) 
                return "SoundConsensus";
              else 
                return "SC"; //short version for small windows
            });

      //append year to header
      header.append("text")
            .attr("class","year")
            .attr("dy", "0.7em")
            .attr("dx", "8em")
            .attr("dx", function() {
              if (width >= 1400) 
                return "8em";
              else 
                return "1.4em"; //condensed version for small windows
            })
           .text("14");

      //append subtitle to header
      header.append("text")
            .attr("class","subtitle")
            .attr("dy", "2.1em")
            .text(function() {
              if (width >= 1400) 
                return "a visual summary of music review scores";
              else 
                return "music review scores"; //short version for small windows
            });
         
      //append artist column head to header
      header.append("text")
            .attr("text-anchor", "end")
            .attr("class","artist")
            .attr("dy", "0.9em")
            .text("Artist")
            .style("pointer-events", "none")
            .attr("transform", function() { 
              return "translate(" + (3.25 * cell_width) + ",0)"; 
            });

      //append album column head to header
      header.append("text")
            .attr("class","album")
            .attr("text-anchor", "end")
            .attr("dy", "2em")
            .text("Album")
            .style("pointer-events", "none")
            .attr("transform", function() { 
              return "translate(" + (3.25 * cell_width) + ",0)"; 
            });

      //append aoty column head to header
      header.append("a")
            .attr("xlink:href", 
              "http://www.albumoftheyear.org/ratings/overall/2014/15")
            .append("text")
            .attr("dy", "0.9em")
            .text("AoTY")
            .attr("class","column")
            .style("fill", "#de2d26")
            .attr("transform", function() { 
              return "translate(" + (3.5 * cell_width) + ",0)"; 
            })
            .append("title")
              .text("albumoftheyear.org");   

      //append AoTY subtitle to header
      header.append("a")
            .attr("xlink:href", 
              "http://www.albumoftheyear.org/ratings/overall/2014/15")
            .append("text")
            .attr("class","album")
            .attr("dy", "2em")
            .text("(overall)")
            .attr("transform", function() { 
              return "translate(" + (3.5 * cell_width) + ",0)"; 
            })
            .append("title")
              .text("albumoftheyear.org");         

      //append column heads to header, one for each dimension
      header.selectAll("column")
            .data(dimensions)
            .enter()
            .append("g")
            .append("a")
            .attr("xlink:href", function(d) {
              return getReviewerData(d,2);
            })
            .append("text")
            .attr("class","column")
            .attr("dy", "1.2em")            
            .attr("transform", function(d,i) { 
              return "translate(" + (4.5 * cell_width + i * cell_width) + ",0)"; 
            })
            .text(function(d) { 
              return d; 
            })
            .on("mouseover", function(d,i) { //specify tooltip behaviour, repurpose tooltip
              d3.select("#tooltip_artist") //reviewer name
                .transition()
                .text(getReviewerData(d,3));
              d3.select("#tooltip_artist_link") //reviewer aoty url
                .transition()
                .attr("xlink:href", getReviewerData(d,1));
              d3.select("#tooltip_album") //reviewer abbreviation
                .transition()
                .text(d);
              d3.select("#tooltip_album_link")
                .transition()
                .attr("xlink:href", getReviewerData(d,1));
              d3.select("#tooltip_genre")
                .transition()
                .text("");
              d3.select("#tooltip_release")
                .transition()
                .text("Founded: " + getReviewerData(d,4));
              d3.select("#tooltip_score")
                .transition()
                .text("Location: " + getReviewerData(d,5));
              d3.select("#tooltip_website_link")
                .transition()
                .attr("xlink:href", getReviewerData(d,2));
              d3.select("#tooltip_website")
                .transition()
                .text(getReviewerData(d,2));
            })
            .append("title")
              .text(function(d) {
                return getReviewerData(d,3);
            });

      /**

      MAIN BODY

      **/     

      //append "table" of rows containing data to main panel
      var table = main_svg.append("g")
                          .attr("class","table")
                          .attr("transform", function(d, i) { 
                            return "translate(0," + 35 + ")"; 
                          });

      var selected_row = null;                    

      //append rows to the table, one for each datum
      var row = table.selectAll("row")
                     .data(data)
                     .enter()
                     .append("g")
                     .attr("class", "row")
                     .on("mouseover", function(d,i) { //specify tooltip behaviour 
                      d3.select("#tooltip_artist")
                        .transition()
                        .text(d.Artist);
                      d3.select("#tooltip_artist_link")
                        .transition()
                        .attr("xlink:href", d.Profile_url);
                      d3.select("#tooltip_album")
                        .transition()
                        .text(d.Album);
                      d3.select("#tooltip_album_link")
                        .transition()
                        .attr("xlink:href", d.Album_url);
                      d3.select("#tooltip_genre")
                        .transition()
                        .text("Genre: " + d.Genre);
                      d3.select("#tooltip_release")
                        .transition()
                        .text("Released: " + d.ReleaseDate + " (" + d.Label + ")");
                      d3.select("#tooltip_score")
                        .transition()
                        .text("AoTY rank (score): " + (i + 1) + " (" + d.AoTY + ")");
                      d3.select("#tooltip_website_link")
                        .transition()
                        .attr("xlink:href", d.Artist_url);
                      d3.select("#tooltip_website")
                        .transition()
                        .text(d.Artist_url);
                    })
                     .on("mouseenter", function(d,i) {
                      if (selected_row != d.Album_url) {             
                        d3.select('.table').selectAll(".row").sort(function (a, b) { // select the parent and sort the path's
                            if (a.Album_url != d.Album_url && a.Album_url != selected_row) return -1;               // a is not the hovered element, send "a" to the back
                            else return 1;                             // a is the hovered element, bring "a" to the front
                        });
                        d3.select(this) //highlight corresponding row of cells 
                          .selectAll("rect")
                          .transition()
                          .duration(200)
                          .style("stroke", "#de2d26");
                        d3.select(this)
                          .selectAll("rect.value")
                          .transition()
                          .duration(200)
                          .style("fill", "#fcbba1")
                          .style("stroke", "#de2d26");
                        d3.select(this)
                          .selectAll("text.album")
                          .style("fill", "#de2d26");
                        d3.select(this)
                          .selectAll("text.artist")
                          .style("fill", "#de2d26");
                        d3.select(this)
                          .selectAll("line.link_line")
                          .style("opacity", "1")
                          .style("stroke", "#de2d26");
                      }
                    })
                  .on("click", function(d,i) { //specify tooltip behaviour
                      if (selected_row == null) {
                        selected_row = d.Album_url;
                        d3.select('.table').selectAll(".row").sort(function (a, b) { // select the parent and sort the path's
                            if (a.Album_url != d.Album_url && a.Album_url != selected_row) return -1;               // a is not the hovered element, send "a" to the back
                            else return 1;                             // a is the hovered element, bring "a" to the front
                        });
                        d3.select(this) //highlight corresponding row of cells 
                          .selectAll("rect")
                          .transition()
                          .duration(200)
                          .style("stroke", "#54278f");
                        d3.select(this)
                          .selectAll("rect.value")
                          .transition()
                          .duration(200)
                          .style("fill", "#bcbddc")
                          .style("stroke", "#54278f");
                        d3.select(this)
                          .selectAll("text.album")
                          .style("fill", "#54278f");
                        d3.select(this)
                          .selectAll("text.artist")
                          .style("fill", "#54278f");
                        d3.select(this)
                          .selectAll("line.link_line")
                          .style("opacity", "1")
                          .style("stroke", "#54278f");
                      }
                      else if (selected_row == d.Album_url){
                        selected_row = null;
                        d3.select(this) //highlight corresponding row of cells 
                          .selectAll("rect")
                          .transition()
                          .duration(200)
                          .style("stroke", "#bbb");
                        d3.select(this)
                          .selectAll("rect.value")
                          .transition()
                          .duration(200)
                          .style("fill", "#ccc")
                          .style("stroke", "#bbb");
                        d3.select(this)
                          .selectAll("text.album")
                          .style("fill", "#666");
                        d3.select(this)
                          .selectAll("text.artist")
                          .style("fill", "#000");
                        d3.select(this)
                          .selectAll("line.link_line")
                          .style("opacity", "1")
                          .style("stroke", "#bbb");
                      }
                    })
                    .on("mouseleave", function(d,i) {  //undo mouseenter events                    
                      if (selected_row != d.Album_url) {                      
                        d3.select(this)
                          .selectAll("rect")
                          .transition()
                          .delay(100)
                          .duration(200)
                          .style("z-index", "0")
                          .style("stroke", "#bbb");
                        d3.select(this)
                          .selectAll("rect.value")
                          .transition()
                          .delay(100)
                          .duration(200)
                          .style("fill", "#ccc")
                          .style("stroke", "#bbb");
                        d3.select(this).selectAll("text.album")
                          .style("fill", "#666");
                        d3.select(this).selectAll("text.artist")
                          .style("fill", "#000");
                        d3.select(this)
                          .selectAll("line.link_line")
                          .style("opacity", "0.25")
                          .style("stroke", "#bbb");
                      }
                    });

      //append row header to each row to contain artist and album
      var row_header = row.append("g")
                          .attr("class","row_header");  

      //append artist link and name to row header
      row_header.append("a")
                .attr("xlink:href", function(d){ 
                  return d.Profile_url ; 
                })     
                .append("text")
                .attr("text-anchor", "end")
                .attr("class","artist")
                .attr("dy", "0.9em")
                .text(function(d) { 
                  return d.Artist; 
                })                            
                .attr("y", function(d,i) {
                  return y(i);
                })
                .attr("transform", function(d, i) { 
                  return "translate(" + (3.25 * cell_width) + ",0)"; 
                });

      //append album link and name to row header
      row_header.append("a")
                .attr("class", "link")
                .attr("xlink:href", function(d) { 
                  return d.Album_url; 
                })
                .append("text")
                .attr("class","album")
                .attr("text-anchor", "end")
                .attr("dy", "2em")
                .text(function(d) { 
                  return d.Album ; 
                })
                .attr("y", function(d,i) {
                  return y(i);
                })
                .attr("transform", function(d, i) { 
                  return "translate(" + (3.25 * cell_width) + ",0)"; 
                });

      //append index to row header
      row_header.append("text")
                .attr("class","index")
                .attr("text-anchor", "end")
                .attr("dy", "1.5em")
                .text(function(d, i) {
                  if (width >= 1400) 
                    return i + 1;
                  else 
                    return ""; //don't show in small windows
                })
                .attr("y", function(d,i) {
                  return y(i);
                })
                .attr("transform", function(d, i) { 
                  return "translate(" + (3.47 * cell_width) + ",0)"; 
                });          

      //append aoty cell to each row
      var aotycell = row.append("g")
                        .attr("class","cell")
                        .attr("y", function(d,i) {
                          return y(i);
                        })
                        .attr("transform", function(d) { 
                          return "translate(" + 
                            (3.5 * cell_width) + 
                            ",0)"; 
                        })
                        .attr("width", cell_width / 1.5)
                        .attr("height", cell_height);

      //append rectangular bounds to each cell
      aotycell.append("rect")   
              .attr("class", "bounds")
              .attr("height", cell_height)
              .attr("width", cell_width / 1.5)
              .attr("y", function(d,i) {
                  return y(i);
              });                 

      //append link to album page and bar scaled to score to cell
      aotycell.append("rect")
              .attr("class","value")
              .attr("height", cell_height)
              .attr("width", function(d) { 
                return z(d.AoTY); 
              })
              .attr("y", function(d,i) {
                return y(i);
              });

      //append album score in the cell
      aotycell.append("text")
              .attr("class","score")
              .attr("height", cell_height)
              .attr("dy", "1.5em")
              .attr("dx", "0.3em")
              .text(function(d) { 
                return d.AoTY; 
              })
              .attr("y", function(d,i) {
                  return y(i);
              });       

      //append line to cell, link to next cell
      aotycell.append("line")
              .attr("class","link_line")
              .attr("x1", function() {
                return x(dimensions[0]) - 3.825 * cell_width;
              })
              .attr("x2", function() {
                  return x(dimensions[0]) - 3.5 * cell_width;
              })
              .attr("y1", function(d,i) {                
                return y(i) + cell_height / 2;
              })
              .attr("y2", function(d,i) {
                return y(getRank(d.Album_url,dimensions[0])) + cell_height / 2;
              })
              .style("display", function(d,i){
                if (getRank(d.Album_url,dimensions[0]) == -1)
                  return "none";
                else
                  return "inline";
              });        

      //append cells to each row, map each cell to a dimension
      var cell = row.selectAll("cell")                
                    .data(function(d) { 
                      return dimensions.map(function(k) {
                        return d[k]; 
                      }); 
                    })
                    .enter()
                    .append("g")
                    .attr("class","cell")                
                    .attr("width", cell_width / 1.5)
                    .attr("height", cell_height);

      //append rectangular bounds to each cell
      cell.append("rect")   
          .attr("class", "bounds")
          .attr("height", cell_height)
          .attr("width", cell_width / 1.5)
          .attr("x", function(d,i) {
            return x(dimensions[i]);
          })
          .attr("y", function(d,i) {
            if (d == '')
              return y(data.length - 1);
            else
              return y(getRank(d3.select(this.parentNode.parentNode).datum().Album_url,dimensions[i]));        
          })
          .style("stroke-width", function(d) { 
            if (d == '')
              return 0 + "px"; 
          });                 

      //append bar scaled to score to cell
      cell.append("rect")
          .attr("class","value")
          .attr("height", cell_height)
          .attr("width", function(d) { 
            return z(d); 
          })
          .attr("x", function(d,i) {
            return x(dimensions[i]);
          })
          .attr("y", function(d,i) {
            if (d == '')
              return y(data.length - 1);
            else
              return y(getRank(d3.select(this.parentNode.parentNode).datum().Album_url,dimensions[i]));        
          });

      //append album score in the cell
      cell.append("text")
          .attr("class","score")
          .attr("height", cell_height)
          .attr("dy", "1.5em")
          .attr("dx", "0.3em")
          .text(function(d) { 
            return d; 
          })      
          .style("fill", function(d) { 
            if (d == 100)
              return "#de2d26"; 
          })
          .style("font-weight", function(d) { 
            if (d == 100)
              return "bold"; 
          })
          .attr("x", function(d,i) {
            return x(dimensions[i]);
          })
          .attr("y", function(d,i) {
            if (d == '')
              return y(data.length - 1);
            else
              return y(getRank(d3.select(this.parentNode.parentNode).datum().Album_url,dimensions[i]));
          });

      //append line to cell, link to next cell
      cell.append("line")
          .attr("class","link_line")
          .attr("x1", function(d,i) {
            return x(dimensions[i]) + cell_width / 1.5;
          })
          .attr("x2", function(d,i) {
            if (i + 1 != dimensions.length)
              return x(dimensions[i+1]);
            else
              return x(dimensions[i]) + cell_width / 1.5;
          })
          .attr("y1", function(d,i) {
            return y(getRank(d3.select(this.parentNode.parentNode).datum().Album_url,dimensions[i])) + cell_height / 2;
          })
          .attr("y2", function(d,i) {
            return y(getRank(d3.select(this.parentNode.parentNode).datum().Album_url,dimensions[i+1])) + cell_height / 2;
          })
          .style("display", function(d,i){            
            if (i + 1 == dimensions.length || 
              getRank(d3.select(this.parentNode.parentNode).datum().Album_url,dimensions[i]) == -1 ||
              getRank(d3.select(this.parentNode.parentNode).datum().Album_url,dimensions[i+1]) == -1)
              return "none";
            else
              return "inline";
          });
        
      //listen for dispatch events from genre selector
      dispatch.on("highlight.row", function(genre,label,consensus_lb,consensus_ub) {
        row.selectAll('.album').style("opacity", function(d){
          if ((d.Genre == genre || genre == "( All Genres )") && 
              (d.Label == label || label == "(  All Record Labels )") &&
              (d.SD >= consensus_lb) && (d.SD <= consensus_ub))
            return 1;
          else 
            return 0.25;
        }); 
        row.selectAll('.artist').style("opacity", function(d){
          if ((d.Genre == genre || genre == "( All Genres )") && 
              (d.Label == label || label == "(  All Record Labels )") &&
              (d.SD >= consensus_lb) && (d.SD <= consensus_ub))
            return 1;
          else 
            return 0.25;
        }); 
        row.style("pointer-events", function(d){
          if ((d.Genre == genre || genre == "( All Genres )") && 
              (d.Label == label || label == "(  All Record Labels )") &&
              (d.SD >= consensus_lb) && (d.SD <= consensus_ub))
            return 'inherit';
          else 
            return 'none';
        }); 
        row.sort(function (d, a) { // select the parent and sort the path's
          if ((d.Genre == genre || genre == "( All Genres )") && 
              (d.Label == label || label == "(  All Record Labels )") &&
              (d.SD >= consensus_lb) && (d.SD <= consensus_ub))
            return 1; // a is not the hovered element, send "a" to the back
          else 
            return -1; // a is the hovered element, bring "a" to the front
        });

        row.selectAll('.cell').style("display", function(d){
          if ((d3.select(this.parentNode).datum().Genre == genre || genre == "( All Genres )") && 
              (d3.select(this.parentNode).datum().Label == label || label == "(  All Record Labels )") &&
              (d3.select(this.parentNode).datum().SD >= consensus_lb) && (d3.select(this.parentNode).datum().SD <= consensus_ub))
            return 'inline';
          else 
            return 'none';
        });    
      });

      //function for getting reviewer metdata, given an abbreviation and a metadata dimension index
      function getReviewerData (reviewer,metadata) {

        var index = reviewer_data.map(function(e) { 
          return e.reviewNames; 
        }).indexOf(reviewer);

        switch(metadata) {
          case 0:
            return reviewer_data[index].reviewNames;
            break;
          case 1:
            return reviewer_data[index].aotyUrls;
            break;
          case 2:
            return reviewer_data[index].url;
            break;
          case 3:
            return reviewer_data[index].fullName;
            break;
          case 4:
            return reviewer_data[index].yearFounded;
            break;
          case 5:
            return reviewer_data[index].location;
            break;
        }
      }

      //function for determining rank according to current dimension 
      function getRank (url,dimension) {

        var index = rank_data.map(function(e) { 
          return e.Album_url; 
        }).indexOf(url);

        var dim_index = rank_dimensions.indexOf(dimension);

        var rank = rank_data[index][rank_dimensions[dim_index]];

        if (rank - 1 == -1)
          console
        
        return rank - 1;
      }

      /**

      FOOTER

      **/ 

      //append container credits to footer panel
      var footer = footer_svg.append("g")
                             .attr("class","footer");

       //append artist column head to header
      footer.append("text")
            .attr("class","artist")
            .attr("id","more_info")
            .attr("dy", "1.3em")
            .attr("dx", "2.2em")
            .text("More Info")
            .on("mouseover", function() {
              d3.select(this).style("fill","#de2d26");
              d3.select('#info_button').attr("xlink:href","info-hover.png");
            })
            .on("mouseout", function() {
              d3.select(this).style("fill","#000");
              d3.select('#info_button').attr("xlink:href","info.png");
            })
            .on("click", function() {
              if (!about_visible) {
                about_visible = true;
                d3.select('#about_panel').style("display","inline");
              }
              else {
                about_visible = false;
                d3.select('#about_panel').style("display","none");
              }
            });                             

      //append title to footer                       
      footer.append("a")
            .attr("xlink:href", 
              "https://twitter.com/mattbrehmer")
            .append("text")
            .attr("class","attribution")
            .attr("dy", "0.6em")
            .attr("dx", "7.5em")
            .text("by @mattbrehmer");

      //append subtitle to footer
      footer.append("a")
            .attr("xlink:href", 
              "http://www.albumoftheyear.org/ratings/overall/2014/15")
            .append("text")
            .attr("class","attribution")
            .attr("dy", "2.0em")
            .attr("dx", "7.5em")
            .text("data from AoTY / albumoftheryear.org");

      //append subtitle to footer
      footer.append("image")
            .attr("id","info_button")
            .attr("xlink:href","info.png")
            .style("cursor","pointer")
            .attr("width", 16)
            .attr("height", 16)
            .on("mouseover", function() {
              d3.select('#more_info').style("fill","#de2d26");
              d3.select(this).attr("xlink:href","info-hover.png");
            })
            .on("mouseout", function() {
              d3.select('#more_info').style("fill","#000");
              d3.select(this).attr("xlink:href","info.png");
            })
            .on("click", function() {
              if (!about_visible) {
                about_visible = true;
                d3.select('#about_panel').style("display","inline");
              }
              else {
                about_visible = false;
                d3.select('#about_panel').style("display","none");
              }
            })
            .append("title")
            .text("More info");

      d3.select("#filter_div")
        .html("Filter by genre, record label, or level of consensus: ")

      var all_genres = ["( All Genres )"];

      //append genre dropdown to footer, 
      var select_genre = d3.select("#filter_div")
                           .append("select")
                           .on("change", dropdownChange),
          genre_options = select_genre.selectAll("option")
                                      .data(all_genres.concat(genre_scale.domain().sort()));

      //populate genre dropdown with genres 
      genre_options.enter()
                   .append("option")
                   .text(function (d) { 
                    return d; 
                   });     
      
      var all_labels = ["(  All Record Labels )"];

      //append label dropdown to footer, 
      var select_label = d3.select("#filter_div")
                          .append("select")
                          .on("change", dropdownChange),
          label_options = select_label.selectAll("option")
                                    .data(all_labels.concat(label_scale.domain().sort()));

      //populate label dropdown with labels
      label_options.enter()
                   .append("option")
                   .text(function (d) { 
                    return d; 
                   });

      //consensus level options
      var consensus_levels = ["( All Consensus Levels )",
                              "Low Consensus",
                              "Low-Medium Consensus",
                              "Medium Consensus",
                              "Medium-High Consensus",
                              "High Consensus"];             

      //append consensus dropdown to footer, 
      var select_consensus = d3.select("#filter_div")
                               .append("select")
                               .on("change", dropdownChange),
          consensus_options = select_consensus.selectAll("option")
                                              .data(consensus_levels);

      //populate consensus dropdown with consensus levels 
      consensus_options.enter()
                       .append("option")
                       .text(function (d) { 
                        return d; 
                       });

      //whenever an option is selected from the dropdowns, issue a dispatch event 
      function dropdownChange() {
        var selected_genre_index = select_genre.property("selectedIndex"),
            selected_genre = genre_options[0][selected_genre_index].__data__,
            selected_label_index = select_label.property("selectedIndex"),
            selected_label = label_options[0][selected_label_index].__data__,
            selected_consensus_index = select_consensus.property("selectedIndex"),
            consensus_lb,consensus_ub; //consensus upper and lower bounds

            switch(selected_consensus_index) {
              case 0: //all consensus levels
                consensus_lb = d3.min(sd_arr);
                consensus_ub = d3.max(sd_arr);
                break;
              case 1: //low consensus
                consensus_lb = d3.quantile(sd_arr,0.8);
                consensus_ub = d3.max(sd_arr);
                break; 
              case 2: //low-medium consensus
                consensus_lb = d3.quantile(sd_arr,0.6);
                consensus_ub = d3.quantile(sd_arr,0.8);
                break;
              case 3: //middle consensus
                consensus_lb = d3.quantile(sd_arr,0.4);
                consensus_ub = d3.quantile(sd_arr,0.6);
                break;
              case 4: //middle-high consensus
                consensus_lb = d3.quantile(sd_arr,0.2);
                consensus_ub = d3.quantile(sd_arr,0.4);
                break;
              case 5: //high consensus
                consensus_lb = d3.min(sd_arr);
                consensus_ub = d3.quantile(sd_arr,0.2);
                break;
            }

        dispatch.highlight(selected_genre,selected_label,consensus_lb,consensus_ub);
      } 

    }); //end d3.csv load
  }); //end d3.csv load
 }); //end d3.csv load
diff --git a/style.css b/style.css
 body {
  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
 }

 #header_panel {
 	position: absolute;
 	top: 0px;
 	left: 0px;
 	margin: 0px;
 	padding: 10px 10px 0px 10px;
 	background: #eee;
 	border-bottom: 1px solid #999;
 }

 #footer_panel {
 	position: fixed;
 	bottom: 0px;
 	left: 0px;
 	margin: 0px;
 	padding: 10px 10px 0px 10px;
 	background: #eee;
  opacity: 1;
 	border-top: 1px solid #999;
 }

 #tooltip_panel {
  position: fixed;
  margin: 0px;
  bottom: 45px;
  right: 10px;
  padding: 10px 10px 0px 10px;
  border: 1px solid #de2d26;
 }

 #about_panel {
  position: fixed;
  margin: 0px;
  bottom: 45px;
  left: 10px;
  padding: 10px 10px 0px 10px;
  text-align: left; 
  width: 420px; 
  height: 420px;    
  font: 9px sans-serif; 
  background: #fee0d2;  
  border: 1px solid #de2d26;          
  border-radius: 4px;
  z-index: 10;
 }

 #filter_div {
 	position: fixed;
 	bottom: 7px;
 	right: 10px;
 	margin: 0px;
 	padding: 0px;
 	z-index: 10;
 	fill: #666;
  font: 8px sans-serif;
 }

 .background path {
  fill: none;
  stroke: #ddd;
  shape-rendering: crispEdges;
 }

 .bounds {
 	fill: #fff;
 	stroke: #bbb;
 	stroke-width: 1px;
  cursor: pointer;
 }

 .value {
 	fill: #ccc;
  stroke: #bbb;
  stroke-width: 1px;
 	cursor: pointer;
 }

 .row {
  pointer-events: all;
 }

 .cell text {
 	font: 8px sans-serif;
 	fill: #666;
 	-webkit-user-select: none; /* Chrome/Safari */        
 	-moz-user-select: none; /* Firefox */
 	-ms-user-select: none; /* IE10+ */
 	cursor: pointer;
 }

 .link_line {
  stroke: #bbb;
  opacity: 0.25;
  stroke-width: 1px;
 }

 .column {
  fill: black;
  font: 8px sans-serif;
  font-weight: bold;
 	cursor: pointer;
 }

 .column:hover {
  fill: #de2d26;
  text-decoration: underline;
 }

 .year {
 	fill: #de2d26;
 	font-weight: bold;
 	font-family: Palatino, Times, serif;
 }

 .title {
 	font-family: Palatino, Times, serif;
 	font-weight: bold;
 	fill: #666;
 	cursor: default;
 }

 .subtitle {
 	font: 8px sans-serif;
 	cursor: default;
 	fill: #666;
 }

 .attribution {
 	font: 8px sans-serif;
 	fill: #666;
  cursor: pointer;
 }

 .attribution:hover {
 	fill: #de2d26;
 	text-decoration: underline;
 }

 .artist {
 	font: 8px sans-serif;
 	font-weight: bold;
 	cursor: pointer;
 }

 .artist:hover {
 	fill: #de2d26;
 	text-decoration: underline;
 }

 .album {
 	font: 8px sans-serif;
 	fill: #666;
 	cursor: pointer;
 }

 .index {
 	font: 7px sans-serif;
 	fill: #999;
 	cursor: default;
 }

 .album:hover {
 	fill: #de2d26;
 	text-decoration: underline;
 }

 .tooltip {
  position: absolute;	
  text-align: left;	
  width: 190px;	
  height: 100px;		
  padding: 5px 5px 5px 5px;
  font: 8px sans-serif;	
  background: #fee0d2;	
  border: 0px #de2d26;					
  border-radius: 4px;
  z-index: 10;
 }

 .vline {
  stroke: #999;
  stroke-width: 1px; 
 }
diff --git a/thumbnail.png b/thumbnail.png
	require(data.table)

	albumscores.dt = scores.dt[albums.dt,allow.cartesian=T] #combine the data tables
	albumscores.dt = unique(albumscores.dt) #remove duplicate rows

	#remove duplicate columns
	albumscores.dt[,Artist.1:=NULL]
	albumscores.dt[,Album.1:=NULL]

	#coerce classes on columns, remove commas from character columns
	albumscores.dt$ReleaseDate = as.Date(albumscores.dt$ReleaseDate,format = "%B %d, %Y")
	albumscores.dt$url = as.factor(albumscores.dt$url)
	setnames(albumscores.dt,"url","Artist_url")
	albumscores.dt$Label = gsub(",","",albumscores.dt$Label)
	albumscores.dt$Label = as.factor(albumscores.dt$Label)
	albumscores.dt$Genre = gsub(",","",albumscores.dt$Genre)
	albumscores.dt$Genre = as.factor(albumscores.dt$Genre)
	albumscores.dt$Artist = gsub(",","",albumscores.dt$Artist)
	albumscores.dt$Artist = as.factor(albumscores.dt$Artist)
	albumscores.dt$Album = gsub(",","",albumscores.dt$Album)
	albumscores.dt$Album = as.factor(albumscores.dt$Album)

	#sort by artist, album
	setkey(albumscores.dt,Artist,Album)

	# albumscores.dt = albumscores.dt[order(-rank(AoTY))]
	#
	# albumscores.dt = albumscores.dt[1:100]

	albumscores.dt = transform(albumscores.dt, SD=apply(albumscores.dt[,3:21, with = F],1, sd, na.rm = TRUE))

	#duplicate table for storing ranks
	albumranks.dt = albumscores.dt

	#ranking function applied to a numeric column
	convertToRank = function(col) {

	if(is.numeric(col)){
	col = rank(-col,na.last = "keep",ties.method = "min")
	col = findInterval(col, sort(unique(col)))
	}

	return(col)
	}
	#rank within numeric columns and replace with ranks
	albumranks.dt = albumranks.dt[, lapply(.SD, convertToRank)]

	#sort ranked table by artist, album
	setkey(albumranks.dt,Artist,Album)

	#copy rank column from score table to rank table
	albumranks.dt$rank = albumscores.dt$rank

	#remove rank column from score table
	albumscores.dt[,rank:=NULL]

	#remove AoTY rank from rank table (rank = AoTY rank)
	albumranks.dt[,AoTY:=NULL]

	#write to csv
	write.csv(albumranks.dt,file="albumranks.csv",quote=F,row.names=F,na="")
	write.csv(albumscores.dt,file="albumscores.csv",quote=F,row.names=F,na="")
	# write.csv(list_urls.dt,file="list_urls.csv",quote=F,row.names=F,na="")
	#main pre-processing script

	#change year as necessary
	yr = 2014

	source("aoty-table-parse.R")
	source("aoty-scrape.R")
	source("aoty-combine.R")

	# require(data.table)

	#preprocess from previously cleaned csv files

	# list_urls.dt = as.data.table(read.csv(file="aoty-list-urls.csv",header = T))
	# setkey(list_urls.dt,name)
	#
	# ranks.dt = as.data.table(read.csv(file="aoty-w-links.csv",header = T))
	# setkey(ranks.dt,Album_url)
	require(data.table)

	albums.dt = data.table()

	scrapeAlbum = function(url) {

	print(c("scraping: ",url))

	raw.data = readLines(as.character(url),warn=F)

	#locate relevant metadata on page
	artistLoc = sapply('.*span class="item', function(y) grep(y,raw.data)[1])
	albumLoc = sapply('.*span class="fn', function(y) grep(y,raw.data)[1])
	releasedLoc = sapply('RELEASED:', function(y) grep(y,raw.data)[1])
	labelLoc = sapply('LABEL:', function(y) grep(y,raw.data)[1])
	genreLoc = sapply('GENRE:', function(y) grep(y,raw.data)[1])
	websiteLoc = sapply('WEBSITE:', function(y) grep(y,raw.data)[1])

	#clean the html off of the metadata
	artist = gsub(".>(.)</a> - <span class=\"fn\">.*","\\1",raw.data[artistLoc])
	artist = gsub(" *$","",artist) #remove trailing white space
	album = gsub(".<span class=\"fn\">(.)</span></span>.*","\\1",raw.data[albumLoc])
	album = gsub(" *$","",album) #remove trailing white space
	releaseDate = gsub(".RELEASED: </span> (.)</div>","\\1",raw.data[releasedLoc])
	releaseDate = gsub(" *$","",releaseDate) #remove trailing white space
	label = gsub(".LABEL: </span>(.*)</div>","\\1",raw.data[labelLoc])
	label = gsub(" *$","",label) #remove trailing white space

	website = gsub(".WEBSITE:.>(.) <img.</a></div><div style.","\\1",raw.data[websiteLoc])
	website = gsub(" *$","",website) #remove trailing white space

	#genre scraping depends on presence / absence of website
	if (is.na(website)) {
	genre = gsub(".GENRE.>(.)</a>.TAGS*","\\1",raw.data[genreLoc])
	genre = gsub("([A-Z], [A-Z])<.*","\\1",genre)
	}
	if (!is.na(website)) {
	genre = gsub(".GENRE.>(.)</a>.WEBSITE.*","\\1",raw.data[genreLoc])
	website = gsub("^","http://",website)
	}

	genre = gsub(" *$","",genre) #remove trailing white space

	remove(artistLoc,albumLoc,releasedLoc,labelLoc,genreLoc,websiteLoc,raw.data)

	return (data.table(artist,album,releaseDate,label,genre,website,url))

	}

	#scrape album metadata from scores table
	albums.dt = do.call("rbind", lapply(X=scores.dt$Album_url,FUN = scrapeAlbum))
	setnames(albums.dt, c("Artist","Album","ReleaseDate","Label","Genre","url","Album_url"))
	setkey(albums.dt,Album_url)
	require(data.table)

	#scrape the aggregate score table for the specified year
	scrapeTable = function(year){

	url = paste("http://www.albumoftheyear.org/ratings/overall/",year,"/15",sep = "")

	raw.data = readLines(url,warn=F)

	#trim everything but the table
	tableBegin = sapply('<TABLE', function(y) grep(y,raw.data)[1])
	tableEnd = sapply('</TABLE>', function(y) grep(y,raw.data)[1])

	table = raw.data[tableBegin:tableEnd]

	return(table)

	}

	#change X in scrapeTable(X) to specify the year e.g. 2014
	table = scrapeTable(2014)

	table = unique(table) #remove duplicate header rows
	table = table[-26] #remove first duplicate header row

	#remove first list item from header
	header = gsub(".(<tr class=\"white-font\".)</a></td></tr><tr.*","\\1",table[1])

	#trim html from header
	header = gsub("</td>",",",header)
	header = gsub("<td>","",header)
	header = gsub("<tr class=\"white-font\" align=\"center\" bgcolor=\"#58808b\">","",header)
	header = gsub("<a class=\"white\" href=\"","",header,)
	header = gsub("</a>","",header,)
	header = gsub("\">",",",header,)

	header = strsplit(header,",") #split string into list
	colNames = c("rank","AoTY")
	reviewNames = header[[1]][((1:length(header[[1]]))%%2==0)]
	reviewNames = reviewNames[3:length(reviewNames)]
	aotyUrls = header[[1]][((1:length(header[[1]]))%%2==1)]
	aotyUrls = aotyUrls[3:length(aotyUrls)]
	colNames = c(colNames,reviewNames,"Artist","Profile_url","Album","Album_url")

	#remove header from row 1
	table[1] = gsub(".(<tr bgcolor=\"#......\">.)","\\1",table[1])

	#coerce chacacters to numeric
	numericFromChar = function(chr) {

	return (as.numeric(as.character(chr)))

	}

	#parse a single row of the table
	parseRow = function(str) {

	cols = t(as.data.table(strsplit(str,"<td")))
	cols = gsub(".n/a.",NA,cols)
	cols = gsub(".>([0-9]+\\.[0-9])<.","\\1",cols)
	cols = cols[2:length(cols)]
	meta = t(as.data.table(strsplit(cols[3],"<div")))
	cols = as.data.table(t(as.data.table(cols[-3])))
	cols = cols[, lapply(.SD, numericFromChar)]
	artist = gsub(".>(.)</a.*","\\1",meta[2],)
	artist_url = gsub(".a href=\"(./)\">.*","http://albumoftheyear.org\\1",meta[2],)
	album = gsub(".>(.)</a.*","\\1",meta[3],)
	album_url = gsub(".a href=\"(.php)\">.*","http://albumoftheyear.org\\1",meta[3],)

	meta = t(as.data.table(c(artist,artist_url,album,album_url)))

	row = as.data.table(cbind(cols,meta))

	setnames(row,colNames)

	return(row)

	}

	#parse the table
	scores.dt = do.call("rbind", lapply(table[1:length(table)-1], FUN = parseRow))
	setkey(scores.dt,Album_url)

	#remove rows with missing data
	scores.dt = unique(scores.dt[!"NA"])

	#clean review urls
	list_urls.dt = as.data.table(cbind(reviewNames,aotyUrls))
	list_urls.dt$aotyUrls = gsub("/ratings/","http://albumoftheyear.org/ratings/",list_urls.dt$aotyUrls)
	setkey(list_urls.dt,reviewNames)

	remove(colNames,reviewNames,table,header,aotyUrls)
	<link rel = "stylesheet" type="text/css" href="style.css" />
	<body>
	<script src="http://d3js.org/d3.v3.min.js"></script>
	<script type="text/javascript" src="soundConsensus.js"></script>
	</body>
reviewNames	aotyUrls	url	fullName	yearFounded	location
405	http://albumoftheyear.org/ratings/18-the-four-oh-five-highest-rated/2014/1	http://www.thefourohfive.com/	The 405	2008	United Kingdom
A.V.	http://albumoftheyear.org/ratings/12-av-club-highest-rated/2014/1	http://www.avclub.com/music/	The A.V. Club	1993	Chicago
AMG	http://albumoftheyear.org/ratings/8-all-music-highest-rated/2014/1	http://www.allmusic.com/	All Music	1991	San Fransisco/Ann Arbor
CoS	http://albumoftheyear.org/ratings/14-consequence-of-sound-highest-rated/2014/1	http://consequenceofsound.net/	Consequence of Sound	2007	Chicago
DiS	http://albumoftheyear.org/ratings/3-drowned-in-sound-highest-rated/2014/1	http://drownedinsound.com/	Drowned in Sound	2000	United Kingdom
DIY	http://albumoftheyear.org/ratings/25-diy-highest-rated/2014/1	http://diymag.com/	DIY Magazine	2011	London
NME	http://albumoftheyear.org/ratings/9-nme-highest-rated/2014/1	http://www.nme.com/	New Musical Express	1952	London
NoRip	http://albumoftheyear.org/ratings/13-no-ripcord-highest-rated/2014/1	http://www.noripcord.com/	No Ripcord	1999	United Kingdom
OMH	http://albumoftheyear.org/ratings/5-musicomh-highest-rated/2014/1	http://www.musicomh.com/	Music OMH	1999	United Kingdom
P4K	http://albumoftheyear.org/ratings/1-pitchfork-highest-rated/2014/1	http://pitchfork.com/	Pitchfork Media	1995	Chicago
Paste	http://albumoftheyear.org/ratings/11-paste-highest-rated/2014/1	http://www.pastemagazine.com/	Paste	2002	United States
PM	http://albumoftheyear.org/ratings/7-popmatters-highest-rated/2014/1	http://www.popmatters.com/	PopMatters	1999	International
PMA	http://albumoftheyear.org/ratings/23-pretty-much-amazing-highest-rated/2014/1	http://prettymuchamazing.com/	Pretty Much Amazing
Radar	http://albumoftheyear.org/ratings/17-under-the-radar-highest-rated/2014/1	http://www.undertheradarmag.com/	Under The Radar	2001	Los Angeles
RS	http://albumoftheyear.org/ratings/35-rolling-stone-highest-rated/2014/1	http://www.rollingstone.com/	Rolling Stone	1967	New York City
Skinny	http://albumoftheyear.org/ratings/31-the-skinny-highest-rated/2014/1	http://www.theskinny.co.uk/	The Skinny	2005	Scotland
Spin	http://albumoftheyear.org/ratings/10-spin-highest-rated/2014/1	http://www.spin.com/	Spin Magazine	1985	New York City
TLOBF	http://albumoftheyear.org/ratings/28-the-line-of-best-fit-highest-rated/2014/1	http://www.thelineofbestfit.com/	The Line of Best Fit		London
TMT	http://albumoftheyear.org/ratings/4-tiny-mix-tapes-highest-rated/2014/1	http://www.tinymixtapes.com/	Tiny Mix Tapes	2001	Minneapolis
	body {
	font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
	}

	#header_panel {
	position: absolute;
	top: 0px;
	left: 0px;
	margin: 0px;
	padding: 10px 10px 0px 10px;
	background: #eee;
	border-bottom: 1px solid #999;
	}

	#footer_panel {
	position: fixed;
	bottom: 0px;
	left: 0px;
	margin: 0px;
	padding: 10px 10px 0px 10px;
	background: #eee;
	opacity: 1;
	border-top: 1px solid #999;
	}

	#tooltip_panel {
	position: fixed;
	margin: 0px;
	bottom: 45px;
	right: 10px;
	padding: 10px 10px 0px 10px;
	border: 1px solid #de2d26;
	}

	#about_panel {
	position: fixed;
	margin: 0px;
	bottom: 45px;
	left: 10px;
	padding: 10px 10px 0px 10px;
	text-align: left;
	width: 420px;
	height: 420px;
	font: 9px sans-serif;
	background: #fee0d2;
	border: 1px solid #de2d26;
	border-radius: 4px;
	z-index: 10;
	}

	#filter_div {
	position: fixed;
	bottom: 7px;
	right: 10px;
	margin: 0px;
	padding: 0px;
	z-index: 10;
	fill: #666;
	font: 8px sans-serif;
	}

	.background path {
	fill: none;
	stroke: #ddd;
	shape-rendering: crispEdges;
	}

	.bounds {
	fill: #fff;
	stroke: #bbb;
	stroke-width: 1px;
	cursor: pointer;
	}

	.value {
	fill: #ccc;
	stroke: #bbb;
	stroke-width: 1px;
	cursor: pointer;
	}

	.row {
	pointer-events: all;
	}

	.cell text {
	font: 8px sans-serif;
	fill: #666;
	-webkit-user-select: none; /* Chrome/Safari */
	-moz-user-select: none; /* Firefox */
	-ms-user-select: none; /* IE10+ */
	cursor: pointer;
	}

	.link_line {
	stroke: #bbb;
	opacity: 0.25;
	stroke-width: 1px;
	}

	.column {
	fill: black;
	font: 8px sans-serif;
	font-weight: bold;
	cursor: pointer;
	}

	.column:hover {
	fill: #de2d26;
	text-decoration: underline;
	}

	.year {
	fill: #de2d26;
	font-weight: bold;
	font-family: Palatino, Times, serif;
	}

	.title {
	font-family: Palatino, Times, serif;
	font-weight: bold;
	fill: #666;
	cursor: default;
	}

	.subtitle {
	font: 8px sans-serif;
	cursor: default;
	fill: #666;
	}

	.attribution {
	font: 8px sans-serif;
	fill: #666;
	cursor: pointer;
	}

	.attribution:hover {
	fill: #de2d26;
	text-decoration: underline;
	}

	.artist {
	font: 8px sans-serif;
	font-weight: bold;
	cursor: pointer;
	}

	.artist:hover {
	fill: #de2d26;
	text-decoration: underline;
	}

	.album {
	font: 8px sans-serif;
	fill: #666;
	cursor: pointer;
	}

	.index {
	font: 7px sans-serif;
	fill: #999;
	cursor: default;
	}

	.album:hover {
	fill: #de2d26;
	text-decoration: underline;
	}

	.tooltip {
	position: absolute;
	text-align: left;
	width: 190px;
	height: 100px;
	padding: 5px 5px 5px 5px;
	font: 8px sans-serif;
	background: #fee0d2;
	border: 0px #de2d26;
	border-radius: 4px;
	z-index: 10;
	}

	.vline {
	stroke: #999;
	stroke-width: 1px;
	}