\name{hopach}
\alias{hopach}
\title{function to perform HOPACH hierarchical clustering}
\description{
The Hierarchical Ordered Partitioning and Collapsing Hybrid (HOPACH)
clustering algorithm builds a hierarchical tree by recursively
partitioning a data set (e.g., gene expression measurements)
with the PAM algorithm, while ordering
and possibly collapsing clusters at each level. The algorithm uses
the Mean/Median Split Silhouette (MSS) criteria to identify the
level of the tree with maximally homogeneous clusters. It also runs
the tree down to produce a final ordered list of the elements.
}
\usage{
hopach(data, dmat = NULL, d = "cosangle", clusters = "best", K = 15,
kmax = 9, khigh = 9, coll = "seq", newmed = "medsil", mss = "med",
impr = 0, initord = "co", ord = "own", verbose=FALSE)
}
\arguments{
\item{data}{data matrix, data frame or exprSet of gene expression measurements. Typically, each column corresponds to an array, and each row corresponds to a gene. For clustering arrays, the arrays appear in the rows and the genes in the columns. All values must be numeric. Missing values are ignored.}
\item{dmat}{matrix or \code{hdist} object of pair wise distances between all genes (arrays). All values
must be numeric, and missing values are not allowed. If NULL, this matrix is computed
using the metric specified by \code{d}. If a matrix is provided, the user is
responsible for ensuring that the metric used agrees with \code{d}.}
\item{d}{character string specifying the metric to be used for calculating
dissimilarities between variables. The currently available options are
"cosangle" (cosine angle or uncentered correlation distance), "abscosangle"
(absolute cosine angle or absolute uncentered correlation distance),
"euclid" (Euclidean distance), "abseuclid" (absolute Euclidean distance),
"cor" (correlation distance), and "abscor" (absolute correlation distance).
Advanced users can write their own distance functions and add these to the
functions \code{distancematrix()} and \code{distancevector()}.}
\item{clusters}{character string specifying if clusters are to be identified
as the level of the tree with the minimum mean/median split
silhouette (MSS) ("best"), the first level of the tree below which MSS
increases ("greedy"), or not at all ("none").}
\item{K}{positive integer specifying the maximum number of levels in the tree.
Must be 15 or less, due to computational limitations (overflow).}
\item{kmax}{integer between 1 and 9 specifying the maximum number of children at
each node in the tree.}
\item{khigh}{integer between 1 and 9 specifying the maximum number of children
at each node in the tree when computing MSS. Can be different from kmax,
though typically these are the same value.}
\item{coll}{character string specifying how collapsing steps are performed at each
level. The options are "seq" (begin with the closest pair of clusters and
collapse pairs sequentially as long as MSS decreases) and "all" (consider all
pairs of clusters and collapse any that decrease MSS).}
\item{newmed}{character string specifying how to choose a medoid for the new
cluster after collapsing a pair of clusters. The options are "medsil"
(maximizer of medoid based silhouette, i.e.: (a-b)/max(a,b), where
a is distance to medoid and b is distance to next closest medoid),
"nn" (nearest neighbor of mean of two collapsed cluster medoids weighted
by cluster size), "uwnn" (unweighted version of nearest neighbor, i.e.
each cluster - rather than each element - gets equal weight), "center"
(minimizer of average distance to the medoid).}
\item{mss}{character vector specifying what criteria function to use. The options
are "med" (median split silhouette) or "mean" (mean split silhouette). See
details for definition of split silhouettes. The MSS criteria is used to
determine the number of children at each node, to decide what collapsing
should be performed at each level, and to determine the main clusters.}
\item{impr}{number between 0 and 1 specifying the margin of improvement in MSS
needed to accept a collapse step. If (MSS.before - MSS.after)/MSS.before
is less than \code{impr}, then the collapse is not performed.}
\item{initord}{character string specifying how to order the clusters in the initial
level of the tree. The options are "co" (maximize correlation ordering, i.e.
the empirical correlation between distance apart in the ordering and distance
between the cluster medoids) or "clust" (apply hopach with binary splits to
the cluster medoids and use the final level of that tree as the ordering). In
subsequent levels, the clusters are ordered relative to the previous level, so
this initial ordering determines the overall structure of the tree.}
\item{ord}{character string specifying how to order the elements within clusters. This
method is used to create an ordering of all elements at the level of the tree
corresponding to the main clusters. The options are "own" (order based on distance
from cluster medoid with medoid first, i.e. leftmost), "neighbor" (order based on
distance to the medoid of the next cluster to the right), or "co" (maximize
correlation ordering - can be slow for large clusters!).}
\item{verbose}{If \code{TRUE} then verbose output is printed.}
}
\details{
The HOPACH hierarchical clustering algorithm is a hybrid between an agglomerative (bottom
up) and a divisive (top down) algorithm. The HOPACH tree is built from the root node (all
elements) down to the leaf nodes, but at each level collapsing steps are used to unite
similar clusters. In addition, the clusters in each level are ordered with a deterministic
algorithm based on the same distance metric that is used in the clustering. In this way,
the ordering produced in the final level of the tree does not depend on the order of the
data in the original data set (as can be the case with algorithms that have a random
component in their ordering methods). Unlike other hierarchical clustering methods, HOPACH
builds a tree of clusters in which the nodes need not be binary, i.e. there can be more
than two children at each split. The divisive steps of the HOPACH algorithm are performed
using the PAM algorithm described in chapter 2 of Kaufman and Rousseeuw (1990) and the R
package 'cluster'.
The Median (or Mean) Split Silhouette (MSS) criteria is used by HOPACH to (i) determine
the optimal number of children at each node, (ii) decide which pairs of clusters to
collapse at each level, and (iii) identify the first level of the tree with maximally
homogeneous clusters. In each case, the goal is to minimize MSS, which is a measure of
cluster heterogeneity described in http://www.bepress.com/ucbbiostat/paper107/.
In hopach versions <2.0.0, these functions returned the square root of
the usual distance for \code{d="cosangle"}, \code{d="abscosangle"},
\code{d="cor"}, and \code{d="abscor"}. Typically, this transformation makes
the dissimilarity correspond more closely with the norm. In order to
agree with the \code{dist} function, the square root is no longer used
in versions >=2.0.0. See ? distancematrix().
}
\value{A list with the following components:
\item{clustering}{the partitioning or 'main clusters' with the following components
'k' is an integer specifying the number of clusters identified by minimizing MSS.
'medoids' is a vector indicating the rows of \code{data} that are the 'k'
cluster medoids, i.e. profiles (or centroids) for each cluster.
'sizes' is a vector containing the 'k' cluster sizes.
'labels' is a vector containing the main cluster labels for every variable. Each
label consists of one digit per level of the tree (up to the level identified as
the main clusters). The digit (1-9) indicates which child cluster the variable
was in at that level. For example, '124' means the fist (leftmost in the tree)
cluster in level 1, the second child of cluster '1' in level 2, and the fourth
child of cluster '12' in level 3. These can be mapped to the numbers 1:k for
simplicity, though the tree structure and relationship amongst the clusters is
then lost, e.g. 1211 is closer to 1212 than to 1221.
'order' is a vector containing the ordering of variables within the main clusters.
The clusters are ordered deterministically as the tree is built. The elements within
each of the main clusters are ordered with the method determined by the value of
\code{ord}: "own" (relative to own medoid), "neighbor" (relative to next medoid
to the right), or "co" (maximize correlation ordering).
}
\item{final}{the final level of the hierarchical tree with the following components
'labels' is a vector containing the final labels for every variable. Each label
consists of one digit per level of the tree (up to the final level), and the
format for the labels is the same as for the clustering labels. The final labels
contain the entire history of the tree. In fact, internal level 'n' can be
reproduced by truncating the final labels to 'n' digits. Ordering the final
labels produces the final ordering (final level of the tree), while ordering
internal level labels produces an ordering of the clusters at that level.
'order' is a vector containing the ordering of variables at the final level of
the tree. Essentially, this is the numeric ordering of the final labels. Due to
the limit on the largest possible integer (overflow), the final labels can have
at most 16 digits, i.e. the tree can have at most 16 levels. For large data sets,
this may not be enough partitioning steps to result in final nodes (leaves) with
only one variable each. Furthermore, PAM can not partition a node of size 3 or
less, so that leaves may contain 2 or 3 variables regardless of the number of
levels in the tree. Hence, the final ordering of variables is completed by
ordering the variables in any leaf of size 2 or larger with the method determined
by the value of \code{ord}: "own" (relative to own medoid), "neighbor" (relative
to next medoid to the right), or "co" (maximize correlation ordering).
'medoids' is a matrix containing the labels and corresponding medoids for each
internal node and leaf of the tree. The number of digits in the label indicates
the level for that node. The medoid refers to a row of \code{data}
}
\item{call}{the matched 'call' generating the HOPACH output}
\item{metric}{the distance metric}
}
\references{
van der Laan, M.J. and Pollard, K.S. A new algorithm for hybrid hierarchical clustering with visualization and the bootstrap. Journal of Statistical Planning and Inference, 2003, 117, pp. 275-303.
\url{http://www.stat.berkeley.edu/~laan/Research/Research_subpages/Papers/hopach.pdf}
\url{http://www.bepress.com/ucbbiostat/paper107/}
\url{http://www.stat.berkeley.edu/~laan/Research/Research_subpages/Papers/jsmpaper.pdf}
Kaufman, L. and Rousseeuw, P.J. (1990). Finding Groups in Data: An Introduction to Cluster Analysis. Wiley, New York.
}
\author{Katherine S. Pollard <kpollard@gladstone.ucsf.edu> and Mark J. van der Laan <laan@stat.berkeley.edu>, with Greg Wall}
\note{Thank you to Karen Vranizan <vranizan@uclink.berkeley.edu> for her input
}
\seealso{\code{\link{distancematrix}}, \code{\link{labelstomss}}, \code{\link{boothopach}}, \code{pam}, \code{makeoutput}}
\examples{
#25 variables from two groups with 3 observations per variable
mydata<-rbind(cbind(rnorm(10,0,0.5),rnorm(10,0,0.5),rnorm(10,0,0.5)),cbind(rnorm(15,5,0.5),rnorm(15,5,0.5),rnorm(15,5,0.5)))
dimnames(mydata)<-list(paste("Var",1:25,sep=""),paste("Exp",1:3,sep=""))
mydist<-distancematrix(mydata,d="cosangle") #compute the distance matrix.
#clusters and final tree
clustresult<-hopach(mydata,dmat=mydist)
clustresult$clustering$k #number of clusters.
dimnames(mydata)[[1]][clustresult$clustering$medoids] #medoids of clusters.
table(clustresult$clustering$labels) #equal to clustresult$clustering$sizes.
#faster, sometimes fewer clusters
greedyresult<-hopach(mydata,clusters="greedy",dmat=mydist)
#only get the final ordering (no partitioning into clusters)
orderonly<-hopach(mydata,clusters="none",dmat=mydist)
#cluster the columns (rather than rows)
colresult<-hopach(t(mydata),dmat=distancematrix(t(mydata),d="euclid"))
}
\keyword{cluster}
\keyword{multivariate}