diff.tex

% This is a template for Ph.D. dissertations in the UCI format.
% 
% All fonts, including those for sub- and superscripts, must be 10
% points or larger.  Recommended sizes are 14-point for chapter
% headings, 12-point for the main body of text and figure/table
% titles, and 10-point for footnotes, sub- and super-scripts, and text
% in figures and tables.
%
% Notes: Add short title to figures, sections, via square brackets,
% e.g. \section[short]{long}.
%
\documentclass[12pt,fleqn]{ucithesis}

% A few common packages
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{array}
\usepackage{graphicx,subfigure}
\usepackage{natbib}
\usepackage{relsize}

% Some other useful packages
\usepackage{caption}
%\usepackage{subcaption}  % \begin{subfigure}...\end{subfigure} within figure
\usepackage{multirow} 
\usepackage{tabularx}
\usepackage{nameref}

\usepackage{epstopdf}
\usepackage{booktabs}
\usepackage{enumerate}
\usepackage{longtable}
\usepackage{array}
\usepackage[table]{xcolor}
\usepackage{hhline}

% plainpages=false fixes the "duplicate ignored" error with page counters
% Set pdfborder to 0 0 0 to disable colored borders around PDF hyperlinks
\usepackage[plainpages=false,pdfborder={0 0 0}]{hyperref}

% Uncomment the following two lines to use the algorithm package,
% which provides an algorithm environment similar to figure and table
% ("\begin{algorithm}...\end{algorithm}"). A list of algorithms will
% automatically be added in the preliminary pages. Note that you
% probably want a package for the actual code to go with this (e.g.,
% algorithmic).
%\usepackage{algorithm}
%\renewcommand{\listalgorithmname}{\protect\centering\protect\Large LIST OF ALGORITHMS}

% Uncomment the following line to enable Unicode support. This will allow you
% to enter non-ASCII characters (such as accented characters) directly without
% having to use LaTeX's awkward escape syntax (e.g., \'{e})
% NOTE: You may have to install the ucs.sty package for this to work. See:
% http://www.unruh.de/DniQ/latex/unicode/
%\usepackage[utf8x]{inputenc}

% Uncomment the following to avoid "widowing", where page breaks cause
% single lines of paragraphs to float onto the next page (this is not
% a UCI requirement but more of an aesthetic choice).
%\widowpenalty=10000
%\clubpenalty=10000

% Modify or extend these at will.
\newtheorem{theorem}{{\sc Theorem}}[chapter]
\newtheorem{definition}{{\sc Definition}}[chapter]
\newtheorem{example}{{\sc Example}}[chapter]

% Uncomment the following to have numbered subsubsections (by default
% numbering goes only to subsections).
%\setcounter{secnumdepth}{4}


% Set this to only select a subset of the includes directives below.
% Very handy to speed up compilation if you're working on a certain
% part of your thesis. It conserves page numbers, references, etc.
% even for non-included files.

\makeatletter
\def\@cline#1-#2\@nil{%
  \omit
  \@multicnt#1%
  \advance\@multispan\m@ne
  \ifnum\@multicnt=\@ne\@firstofone{&\omit}\fi
  \@multicnt#2%
  \advance\@multicnt-#1%
  \advance\@multispan\@ne
  \leaders\hrule\@height\arrayrulewidth\hfill
  \cr
  \noalign{\nobreak\vskip-\arrayrulewidth}}
\makeatother
%DIF PREAMBLE EXTENSION ADDED BY LATEXDIFF
%DIF UNDERLINE PREAMBLE %DIF PREAMBLE
\RequirePackage[normalem]{ulem} %DIF PREAMBLE
\RequirePackage{color}\definecolor{RED}{rgb}{1,0,0}\definecolor{BLUE}{rgb}{0,0,1} %DIF PREAMBLE
\providecommand{\DIFaddtex}[1]{{\protect\color{blue}\uwave{#1}}} %DIF PREAMBLE
\providecommand{\DIFdeltex}[1]{{\protect\color{red}\sout{#1}}}                      %DIF PREAMBLE
%DIF SAFE PREAMBLE %DIF PREAMBLE
\providecommand{\DIFaddbegin}{} %DIF PREAMBLE
\providecommand{\DIFaddend}{} %DIF PREAMBLE
\providecommand{\DIFdelbegin}{} %DIF PREAMBLE
\providecommand{\DIFdelend}{} %DIF PREAMBLE
%DIF FLOATSAFE PREAMBLE %DIF PREAMBLE
\providecommand{\DIFaddFL}[1]{\DIFadd{#1}} %DIF PREAMBLE
\providecommand{\DIFdelFL}[1]{\DIFdel{#1}} %DIF PREAMBLE
\providecommand{\DIFaddbeginFL}{} %DIF PREAMBLE
\providecommand{\DIFaddendFL}{} %DIF PREAMBLE
\providecommand{\DIFdelbeginFL}{} %DIF PREAMBLE
\providecommand{\DIFdelendFL}{} %DIF PREAMBLE
%DIF END PREAMBLE EXTENSION ADDED BY LATEXDIFF
%DIF PREAMBLE EXTENSION ADDED BY LATEXDIFF
%DIF HYPERREF PREAMBLE %DIF PREAMBLE
\providecommand{\DIFadd}[1]{\texorpdfstring{\DIFaddtex{#1}}{#1}} %DIF PREAMBLE
\providecommand{\DIFdel}[1]{\texorpdfstring{\DIFdeltex{#1}}{}} %DIF PREAMBLE
%DIF END PREAMBLE EXTENSION ADDED BY LATEXDIFF

\begin{document}

% Preliminary pages are always loaded (TOC, CV, etc.)
\thesistitle{
  Calico: An early-phase deisgn tool to support the design behaviors of software designers at the whiteboard
}

\degreename{Doctor of Philosophy}

% Use the wording given in the official list of degrees awarded by UCI:
% http://www.rgs.uci.edu/grad/academic/degrees_offered.htm
\degreefield{Computer Science}

% Your name as it appears on official UCI records.
\authorname{Nicolas Francisco Mangano}

% Use the full name of each committee member.
\committeechair{Andre' van der Hoek}
\othercommitteemembers
{
  Professor David Redmile\\
  Professor Gary Olson
}

\degreeyear{2013}

\copyrightdeclaration
{
  {\copyright} {\Degreeyear} \Authorname
}

% If you have previously published parts of your manuscript, you must list the
% copyright holders; see Section 3.2 of the UCI Thesis and Dissertation Manual.
% Otherwise, this section may be omitted.
% \prepublishedcopyrightdeclaration
% {
% 	Chapter 4 {\copyright} 2003 Springer-Verlag \\
% 	Portion of Chapter 5 {\copyright} 1999 John Wiley \& Sons, Inc. \\
% 	All other materials {\copyright} {\Degreeyear} \Authorname
% }

% The dedication page is optional.
\dedications
{
  (Optional dedication page)

  To ...
}

\acknowledgments
{
  I would like to thank...

  (You must acknowledge grants and other funding assistance. 

  You may also acknowledge the contributions of professors and
  friends.

  You also need to acknowledge any publishers of your previous
  work who have given you permission to incorporate that work
  into your dissertation. See Section 3.2 of the UCI Thesis and
  Dissertation Manual.)
}


% Some custom commands for your list of publications and software.
\newcommand{\mypubentry}[3]{
  \begin{tabular*}{1\textwidth}{@{\extracolsep{\fill}}p{4.5in}r}
    \textbf{#1} & \textbf{#2} \\ 
    \multicolumn{2}{@{\extracolsep{\fill}}p{.95\textwidth}}{#3}\vspace{6pt} \\
  \end{tabular*}
}
\newcommand{\mysoftentry}[3]{
  \begin{tabular*}{1\textwidth}{@{\extracolsep{\fill}}lr}
    \textbf{#1} & \url{#2} \\
    \multicolumn{2}{@{\extracolsep{\fill}}p{.95\textwidth}}
    {\emph{#3}}\vspace{-6pt} \\
  \end{tabular*}
}

% Include, at minimum, a listing of your degrees and educational
% achievements with dates and the school where the degrees were
% earned. This should include the degree currently being
% attained. Other than that it's mostly up to you what to include here
% and how to format it, below is just an example.
\curriculumvitae
{

\textbf{EDUCATION}

  \begin{tabular*}{1\textwidth}{@{\extracolsep{\fill}}lr}
    \textbf{Doctor of Philosophy in Computer Science} & \textbf{2012} \\
    \vspace{6pt}
    University name & \emph{City, State} \\
    \textbf{Bachelor of Science in Computational Sciences} & \textbf{2007} \\
    \vspace{6pt}
    Another university name & \emph{City, State} \\
  \end{tabular*}

\vspace{12pt}
\textbf{RESEARCH EXPERIENCE}

  \begin{tabular*}{1\textwidth}{@{\extracolsep{\fill}}lr}
    \textbf{Graduate Research Assistant} & \textbf{2007--2012} \\
    \vspace{6pt}
    University of California, Irvine & \emph{Irvine, California} \\
  \end{tabular*}

\vspace{12pt}
\textbf{TEACHING EXPERIENCE}

  \begin{tabular*}{1\textwidth}{@{\extracolsep{\fill}}lr}
    \textbf{Teaching Assistant} & \textbf{2009--2010} \\
    \vspace{6pt}
    University name & \emph{City, State} \\
  \end{tabular*}

\pagebreak

\textbf{REFEREED JOURNAL PUBLICATIONS}

  \mypubentry{Ground-breaking article}{2012}{Journal name}

\vspace{12pt}
\textbf{REFEREED CONFERENCE PUBLICATIONS}

  \mypubentry{Awesome paper}{Jun 2011}{Conference name}
  \mypubentry{Another awesome paper}{Aug 2012}{Conference name}

\vspace{12pt}
\textbf{SOFTWARE}

  \mysoftentry{Magical tool}{http://your.url.here/}
  {C++ algorithm that solves TSP in polynomial time.}

}

% The abstract should not be over 350 words, although that's
% supposedly somewhat of a soft constraint.
\thesisabstract
{
  The abstract of your contribution goes here.
}


%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***

\preliminarypages

% Include the different components of your thesis, in separate files.
 \newpage \chapter{Introduction}
\label{chapter:introduction}

Often when developers are faced with a design challenge, they will turn to the whiteboard.  This is typical during the conceptual stages of software design, when no code is in existence yet, but may also happen when a significant code base has already been developed, for instance, to plan new functionality or discuss optimizing a key component. Design sessions at the whiteboard may even arise spontaneously, such as when a developer has to refactor some code or discuss how to best integrate a new feature.

Compared to the polished, precise, and typically detailed models software designers produce using modern software design tools, the content they create at the whiteboard consists of rough sketches and imprecise approximations of the design they have in mind, which are continuously modified and refined as part of the design activity [3]. The sketches on the whiteboard wall shown in Figure \ref{figure:software-whiteboard} illustrate this point. The sketches are the result of many design sessions at a startup software company. While the sketches clearly are unintelligible to those who were not present during the design sessions, they serve a crucial role for those who were: they were the vehicle for working through a complex design problem and making key decisions that defined the software to be developed.

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/software-whiteboard}
  \caption{Whiteboard wall at a startup company.}
  \label{figure:software-whiteboard}
\end{figure*}

Developers turn to the whiteboard for the flexibility and fluidity that it offers in the design experience \cite{cherubini2007let}. On a whiteboard, developers can freely sketch, branch off to another part of the design problem, return to a previous part, erase some portion of their work, redraw it, and so on, all without the typical restrictions one might find in a traditional software design environment. They can focus on designing without worrying about particular notations, how to navigate within a tool, etc.

While a preferred medium for design, a significant disadvantage of the whiteboard is that it is a passive medium: it has no facilities that purposefully support the design process. Particularly, whatever is drawn or written remains static and cannot be manipulated, other than drawing over it or erasing it. This is a less than desirable situation, because it is known that software designers often wish to manipulate a design at hand in more advanced ways than merely adding or erasing content \cite{dekel2007notation}. The whiteboard, thus, limits what they are able to do, or at least how efficient they might be at doing so.

The research community has acknowledged this problem and has contributed many approaches that rely on an electronic whiteboard to provide more advanced support (e.g., \cite{chen2008sumlow, landay1995interactive, hammond2006ladder, damm2000supporting, chung2005inkkit}). This support can be divided into two broad categories: (1) approaches that focus on sketch recognition, and (2) approaches that focus on management of sketched content. Approaches in the first category, sketch recognition, attempt to interpret the strokes made by the user to turn them into formal  objects. Early work offered a predefined visual vocabulary for converting sketches into formal objects, such as UML diagrams \cite{chen2008sumlow} or user interface mockups \cite{landay1995interactive}, with tools that provided feedback to the designer based on the rules of the formal notation they support. Later work made visual vocabularies expandable by users \cite{hammond2006ladder} and made using the tools more flexible by delaying interpretation until it was desired by the user \cite{damm2000supporting}, sometimes even while retaining a sketchy appearance \cite{chung2005inkkit}.

Approaches in the second category, sketch management, help organize the potentially many and varied sketched artifacts that may be produced during meetings (Figure \ref{figure:software-whiteboard} is an illustration of this point). Early approaches provided access to a large number of whiteboards through a filmstrip \cite{stefik1987beyond}, hyperlinks \cite{Streitz:1994:DIM:192844.193044}, or hierarchical perspectives \cite{newman2003denim}. Later work automated particular aspects of managing sketches by automatically grouping clusters of sketches in close spatial proximity \cite{mynatt1999flatland}, shrinking sketches when moved to the periphery \cite{guimbretiere2001fluid}, or using metaphors such as Post-It Notes to organize and relate sketches \cite{klemmer2001designers}.  

In examining these and other existing sketching tools, it is useful to consider their respective underlying motivations. In so doing, I observe that every sketch tool was designed to support a particular way of working at the whiteboard. For instance, Knight supports designers in refining initial rough sketches into more formal representations \cite{damm2000supporting}. As another example, Flatland supports designers in creating many different diagrams by automatically clustering sketches and adding specialized behaviors to those clusters \cite{mynatt1999flatland}.

In this dissertation, I define these ways of working as design behaviors. More precisely, I define a design behavior as a recurrent, recognizable set of actions serving a single purpose within a design meeting. In the remainder, I scope these actions to pertain to the whiteboard, that is, focus on creating and modifying its content, navigating the content created, and collaborating in all of these. That is, two designers calling up a colleague is not a design behavior in which I am interested.  One designer refining a diagram drawn by another is.

Quite a few design behaviors have been identified in the literature, despite the fact that the study of software designers ``in action'' is still in its infancy. For example, in addition to refinement of sketches and supporting multiple different types of sketches, studies of designers at OOPSLA’s DesignFest found that software designers improvise their own notations and evolve their diagrams across many canvases \cite{dekel2007notation}. As another example, in-the-field observations at software companies found that software designers deliberately switch among formalisms and use provisionality to engage in a dialog with incomplete ideas \cite{petre2009insights}. 

Broadly speaking, design behaviors that software designers perform can be be described as falling into three high level categories. The  first is how they create and modify the kinds of things they draw, such as the different types of sketches they create and their improvisation of notations. The second is how they navigate those sketches, such as shifting focus between sketches of different types. The third is how they collaborate over their sketches, such as when designers switch from working together on a sketch synchronously to working asynchronously. 

The key insight motivating our research is that software designers do not ``operate in'' or apply just one behavior for an entire design meeting. Rather, designers interleave design behaviors over the course of a design meeting, switching among them as they see fit to navigate a design problem and its potential solutions. For instance, a designer may first sketch two diagrams and juxtapose them side-by-side to evolve them in parallel, then record patterns of execution in one of the diagrams using an impromptu notation, and thereafter shift to a different aspect of the design problem altogether. Throughout, the designer fluidly exhibits a range of these different design behaviors, typically without an explicit trigger. Designers shift opportunistically, addressing the design at hand in the way they see most beneficial at that moment.

During a meeting, it is a natural choice for designers to limit themselves to a single tool to support them. That tool is typically a whiteboard or paper \cite{petre2009insights}, though in some cases it may be a computerized tool like the ones I have described above (e.g., SUMLOW \cite{chen2008sumlow}, Knight \cite{damm2000tool}, Flatland \cite{mynatt1999flatland}. In the latter case, the choice of tool determines the behavior or small set of behaviors that are now supported, as designers will seldom move between tools during meetings, because of the high cost associated with switching, both in terms of the cognitive burden on the user to switch contexts and in terms of the effort required to import or manually copy the contents. The cost is simply too great and designers, thus, are stuck with support for at best a few of their behaviors as embedded in the tool they happen to be using.

What is desired is a tool that supports a broad range of behaviors and allows developers to use the features naturally when they need them. Creating such a tool, however, is a non-trivial exercise. Simply picking up functionality from one tool and dropping it in another, and doing this repeatedly to support a multitude of behaviors, leads to tools that are highly disjoint. It is unclear, for instance, what it would mean for a tool to have available both multiple canvases in a filmstrip and functionality that automatically makes room on the current board? As another example, a tool that automatically recognize sketches and also supports emergent notations is equally difficult to envision as being conceptually clear to its users. Existing solutions do not necessarily stack their functionality gracefully, and the approach taken by one tool may collide with the support provided by another.

This dissertation explicitly addresses the interleaving behaviors that designers exhibit during software design at the whiteboard by taking a step back, examining a collection of behaviors, and contributing a new tool that is designed from the ground up to support this collection of behaviors with a small set of conceptually coherent functionalities. In order to put this into practice, I first built a basic sketching interface so that, as a baseline, designers may sketch and perform the same activities as they normally would on a whiteboard, but in a digital medium. Building on this foundation, I incrementally introduced features that support one or more design behaviors, yet minimally obstruct the support for other behaviors. 

This approach was realized by the construction of a software tool called Calico. In this dissertation, I describe two versions of the tool, the first of which is an initial exploration in supporting a subset of the design behaviors with a sketch-based tool, and the second of which significantly iterates on the first to support the complete set of design behaviors I set out to support. 

The first version of Calico supported an initial subset of the design behaviors with three main features. Figure \ref{fig:calico-version-one} depicts the interface for this version of Calico. The first feature, scraps, supported the kinds of sketches that designers created by providing a mechanism to create representations for box-and-arrow type diagrams and for manipulating sketches. The second feature, the grid, supported designers in navigating between sketches by providing a grid-layout of all canvases. Depicted in Figure \ref{fig:calico-version-one-grid}, users can partition their work over the canvases in the grid, and move the resulting canvases around to organize their work. The third feature, the palette, allowed users to save scraps (depicted on the right of Figure \ref{fig:calico-version-one-canvas}), and reuse them by dragging them back onto the canvas. Chapter 3 explains these features in greater detail.

\begin{figure}
  \centering
  \subfigure[Canvas perspective] {
      \label{fig:calico-version-one-canvas}
      \includegraphics[width=8.5cm,keepaspectratio]{./figures/CalicoVersionOneCanvas}
%      \resizebox{.45\hsize}{.35\hsize}{ }
   }
  \subfigure[Grid perspective] {
      \label{fig:calico-version-one-grid}
      \includegraphics[width=6.5cm,keepaspectratio]{./figures/CalicoVersionOneGrid}
%      \resizebox{.45\hsize}{.35\hsize}{ }
   }
   \caption {Calico version one}
   \label{fig:calico-version-one}
\end{figure}

Calico version one was evaluated in a controlled laboratory study in which I compared the design activity of computer science graduate students engaged in a challenging software design problem using Calico against those using a regular whiteboard. I presented all participants with a prompt to design an educational traffic simulator and asked them to either design it using either Calico or the whiteboard. I found that the same design behaviors performed at the whiteboard were also performed in Calico, but in a slightly different manner. The participants in the Calico groups did use Calico's advanced feature set to perform the design behaviors, and reported that they found the features useful. However, the participants rarely used the palette. The design conversation of the groups were also analyzed using protocol analysis, which was done by breaking down the sessions into segmented phrases belonging to different categories of design. The breakdown of the categories demonstrated a very high correlation with reports from past research of how software design was performed at actual software companies, providing evidence that the design as it happened in our study matched that of design ``in the wild''. Lastly, the groups reported that the limitation of the interactive whiteboard used to only accept a single person drawing at a time hindered their ability to perform parallel work.

Following our experiences with Calico Version One, I rebuilt the tool from the ground up in Calico version two. The features in the second version of Calico significantly iterate on those in the first version to support the whole set of fourteen design behaviors I identified as crucial to designers at the whiteboard. This version is depicted in Figure \ref{fig:calico-version-two}. The scraps feature was significantly revised based on user feedback to better support the designers in the kinds of sketches designers create. The grid feature was replaced with intentional interfaces, depicted in Figure \ref{fig:calico-version-two-ii}, in order to better support designers in navigating among projects and the canvases capturing each project. In our experience with the grid, I saw that users typically dedicated entire rows or columns to a single topic. That behavior inspired intentional interfaces, in which canvases are layed out in a circle, and a set of canvases focusing on a particular topic can extend outwards radially (Figure \ref{fig:calico-version-two-ii}). Relationships between associated canvases are explicitly captured as well using light-weight tagging. Lastly, the software architecture of Calico itself was rebuilt using a client-server architecture in order to allow designers on different machines to collaborate on the same sketch. Chapter 5 explains these features, as well as others, in more detail.

\begin{figure}
  \centering
  \subfigure[Canvas perspective] {
      \label{fig:calico-version-two-canvas}
      \includegraphics[width=8.5cm,keepaspectratio]{./figures/CalicoVersionTwo/CalicoVersionTwoCanvas}
%      \resizebox{.45\hsize}{.35\hsize}{ }
   }
  \subfigure[Intentional interfaces perspective] {
      \label{fig:calico-version-two-ii}
      \includegraphics[width=6.5cm,keepaspectratio]{./figures/CalicoVersionTwoCluster}
%      \resizebox{.45\hsize}{.35\hsize}{ } 
   }
   \caption {Calico version two}
   \label{fig:calico-version-two}
\end{figure}

Calico Version Two was evaluated by deploying it at three field sites, including a commercial open source software company, an interaction design company, and a geographically distributed research group. All groups had access to a large electronic whiteboard running a Calico client, as well as pen-based tablets to connect to a shared Calico server. Members of each respective group conducted extensive design sessions using Calico to perform their own work within their respective organization. I collected and reviewed usage logs from the Calico server, and also conducted semi-structured face-to-face interviews. From this data, I evaluated the use of Calico's features in supporting the representations they created, their strategies for navigating between sketches, and how they collaborated on their sketches. 

Overall, the field evaluations gave insight into how Calico supports design in a real world setting. Usage at the commercial open source software company provided insight in how Calico supported a group engaged in detailed coding, with the group at large and individuals using Calico multiple times, and also personal sessions over both software components and software code. Usage at the interaction design group provided insight into how Calico helped them create personas and storyboards, for example, by organizing pictures of those they interviewed in one- and two-dimensional plots. Lastly, usage by the research group provided insight into how Calico supported a set of distributed researchers in a long-term collaboration. Their use included the development of a software system over a period of several months, with intensive sessions during which they designed process flow diagrams to model the behavior of their software. Overall, the four features collectively supported all fourteen of the design behaviors both in ways that I did and did not expect. 

\section{Thesis structure}

This thesis is organized into nine chapters. The remaining chapters are structured as follows:

\textbf{Chapter \ref{chapter:motivation} - \nameref{chapter:motivation}.} This chapter introduces the definition of design behaviors in detail and presents the grand set of design behaviors that I aim to support. The research question of this thesis is also introduced here.

\textbf{Chapter \ref{chapter:research-question} - \nameref{chapter:research-question}.} We pose the topic that this dissertation will address, layout a method by which I approach this topic, and how I will evaluate this approach.

\textbf{Chapter \ref{chapter:calico-version-one} - \nameref{chapter:calico-version-one}.} I describe the approach and features of the first version of Calico. This chapter introduces the first concepts of scraps, the grid, as well as the palette. The approach is evaluated in a comparative study between Calico and the regular whiteboard, both with respect to the design behaviors performed as well as the design conversations that took place.

\textbf{Chapter \ref{chapter:calico-version-two} - \nameref{chapter:calico-version-two}.} The second and final version of Calico is introduced in this chapter. The new features are introduced, including the revised interaction for scraps, the intentional interfaces features that replaces the grid, the distributed nature of the architecture, as well as other features targeted at supporting the full set of design behaviors.

\textbf{Chapter \ref{chapter:evaluation} - \nameref{chapter:evaluation}.} Here I present our evaluation of the final version of Calico ``in the wild'' at a commercial open source company, an interaction design company, and a distributed research group. A comprehensive evaluation was performed across three field sites, examining the use of features to support the representations used, the navigation among the sketches, and the collaboration patterns.

\textbf{Chapter \ref{chapter:discussion} - \nameref{chapter:discussion}.} In this chapter, I take a step back and put my work in a broader context. I discuss the implications of my findings, reconnect my work to the broader design literature, and provide some findings and observations that are outside the evaluative framework of the design behaviors, but are worthwhile to highlight nonetheless. 

\textbf{Chapter \ref{chapter:related-work} - \nameref{chapter:related-work}.} I review the existing literature of tools that support software design on an electronic whiteboard, covering both tools that support sketch recognition and those that support the management of sketches.

\textbf{Chapter \ref{chapter:conclusions} - \nameref{chapter:conclusions}.} This chapter summarizes the contributions of my dissertation, and also provides some concluding remarks, and offers an outlook at future work. I particularly suggest the following future avenues of research : (1) the concept of compositional notations, which builds on scraps to provide more targeted support for software design notations, (2) informal analysis of scraps in order to provide intelligent feedback on designs in progress such as critiquing the performance of an activity diagram, (3) contextualized sketching, which may import artifacts from an existing code repository to help with design discussions in Calico, and (4) design histories, which may help with reflecting on design activities performed.

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 
 \newpage \chapter{Motivation}
\label{chapter:motivation}

\section{Design Behaviors}

Before introducing the design behaviors that I aim to address in my research, it is useful to firmly define a design behavior. I define a design behavior as \emph{a recurrent, recognizable set of actions serving a single purpose within a design meeting.} This definition recognizes the breadth of activities that may occur during a design meeting, from phoning a colleague, to using sticky notes to brainstorm, to sketching some concepts.  For purposes of this dissertation, however, we scope our investigation to design behaviors that occur at the whiteboard, and specifically those pertaining to creating and modifying whiteboard content, navigating the content created, and collaborating in doing so.

I make four important observations regarding this definition:

\begin{itemize}
\item \textbf{\emph{recurrent}} -- A design behavior can be observed to happen consistently across many design meetings and across many designers. Performing a particular action once in a meeting, such as drawing a diagram of an intersection, does not make 'intersection drawing' a design behavior.  Repeated behavior, particularly as applied to different types of diagrams, across multiple design meetings is required. 
\item \textbf{\emph{recognizable}} -- A design behavior stands out as a coherent set of actions within an overall design meeting. The actions clearly belong together and can be distinguished from other groups of actions.
\item \textbf{\emph{set}} -- A design behavior necessarily unfolds over time with multiple actions that a designer undertakes. A single stroke or gesture does not qualify. 
\item \textbf{\emph{serving a single purpose}} -- A design behavior has a purpose in the overall exploration of a design problem and its potential solutions. The set of actions contribute to furthering the exploration.  
\end{itemize}

With this definition in hand, I examined both the software design literature and the broader design literature for design behaviors. What I found was that the general design literature is more mature than the software design literature, in that it has identified quite a few design behaviors that span across different design disciplines. For instance, designers across building architecture, engineering, and product design use constraints to guide their design thinking \cite{cross2007designerly}. As another example, designers in multiple fields use visual similarity between their sketches and the final product to help them envision the eventual physical artifact \cite{do1998right}.

The software design literature is only now beginning to catch up to the topic of design behaviors, with the emergence of studies that are beginning to look at software designers ``in action'' (e.g., \cite{baker2012guest,cherubini2007let,dekel2007notation,petre2009insights}). These studies, thus far, confirm the behaviors seen in other design disciplines, but at the same time do not confirm all of them yet, simply because the number of studies remains small. In the below, I include a subset of the design behaviors found in the general design literature, quite a few of which have already been confirmed in the software design literature (i.e., Behaviors 1, 2, 3, 4, 6, 7, 8, 9, 10, and 13) and some of which I believe will be confirmed in future (i.e., Behaviors 5, 11, 12, and 14). I feel confident in making this assumption because of my own informal observations of software designers in action, particularly in studying the videos of the SPSD 2010 workshop \cite{baker2012guest}. While I have not performed full studies of those videos expressly for the purpose of corroborating the general design literature, informally I have seen multiple instances of all of the behaviors I discuss in the below. 

I separate the fourteen behaviors I intend to support with Calico into three categories:

 \begin{enumerate}
 \item Kinds of sketches software designers produce
 \item How they use the sketches to navigate through a design problem
 \item How they collaborate on them
 \end{enumerate}

Each of these categories I detail below.

\section{Kinds of sketches software designers produce}
\label{chapter:motivation:kinds}

The first category deals with what the designers draw. The sketches that designers make at the whiteboard are typically not the goal in and of themselves and, as such, the types of sketches that designers make will vary depending on their current design activity. Sometimes, the sketches are used to help the designer in their thinking, by externalizing their ideas and thoughts onto the whiteboard \cite{lawson1994design}. Other times, the whiteboard is simply a medium to explain a thought, idea, or design in progress. The developer is not using it for problem solving, but instead to communicate information to a listener or collaborator \cite{eugene1992engineering}. Because of these different purposes, what designers draw is dependent on what they work on at what point during a design meeting. This leads to the following design behaviors.

\begin{enumerate}
\item \emph{They draw different kinds of diagrams.} In order to explore a design problem, software designers sketch many different types of diagrams, often within the same canvas [5,17]. They may sketch, for instance, entities and relationships, interface mockups, scenarios, architectures, and other kinds of diagrams \cite{cherubini2007let}. The freedom to sketch multiple kinds of diagrams in the same space is fundamental to supporting the exploration of the design space, as it enables designers to explore an issue from different angles, at different levels of abstraction, or even in different ways altogether. The sketches in Figure \ref{figure:motivation-designbehavior-kinds}, which contains a snapshot of the whiteboard from a design session in which two software designers are designing an intersection in a traffic simulator, present such an example. On the right, the designers used the sketch of a traffic intersection to help them in working through and refining the UML model of some of the entities in the simulation on the left. In turn, their work on the entities lead to revisions to the sketch of the intersection on the right. This behavior of working with multiple representations aligns well with the observed phenomenon that tools which restrict designers to use one notation hinder the design exploration and lead to fewer alternatives being considered \cite{shipman1999incremental}. 

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=8cm,keepaspectratio]{./figures/motivation-designbehavior-kinds.png}
  \caption{Diagram from a design session that includes UML diagrams, a map, and annotations.}
  \label{figure:motivation-designbehavior-kinds}
\end{figure*} 


\item \emph{They produce sketches that draw what they need, and no more.} Of the many sketches that software designers create, few are drawn in full detail. Software designers typically can get what they need from a quick and incomplete sketch, e.g., a barebones user interface or boxes-and-arrows sketch \cite{virzi1996usability}. The benefit of a low-detail sketch is that it can be created quickly, and modified easily, giving the designer more rapid feedback \cite{cherubini2007let,petre2009insights}. Further, providing too much structure too soon can create unconscious barriers to change, resulting in a less exploratory and a less broad search for candidate solutions \citep{wong1992rough}. This behavior further breaks down into two parts:
 \begin{enumerate}
 \item \emph{They only draw what they need with respect to the design at hand.} Low-detail sketches tend to “incorporate relevant information and omit the irrelevant” \cite{tversky2002sketches}, including only as much detail as necessary to advance the designers' thinking. When talking, they will only draw what they need to reinforce what they want to communicate \cite{petre2009insights}. When thinking, they will only draw the details relevant to the immediate issue to help them reason \citep{dekel2007notation}. For example, the elements in the left of Figure \ref{figure:motivation-designbehavior-kinds} differ in the amount of detail they contain, where some contain data attributes and others contain only a name.
 \item \emph{They use only those notational conventions that suit drawing what they need.} Sketches only include as much notational convention as the designer needs in a given situation \cite{petre2009insights}. For instance, if a sketch can express an idea using only boxes-and-arrows, then no more will be drawn, but if a sketch must represent a hierarchical relation, then a richer array of arrows will be present, typically following the convention of an existing formal notation.
 \end{enumerate}

\item \emph{Over time, they refine and evolve their sketches.} The level of detail that designers want in their sketches varies over time. Early on, they may need very little, but later they may need much more as they expand on their ideas \citep{ossher12flexible}. Figure \ref{figure:motivation-designbehavior-refinement} presents an example of such a refinement, which details the evolution of the diagrams from Figure \ref{figure:motivation-designbehavior-kinds} from a relatively simple list, to a complex box-and-arrow diagram. As designers discuss and work through an idea, they will evolve their sketches with additional details to capture their decisions. In general, as part of this refinement process, the sketches will contain more visual precision, and the designer will rework the design to fix any inconsistencies \citep{damm2000supporting}. This behavior, too, breaks down into two parts:
 \begin{enumerate}
 \item	\emph{They detail their sketches with increasing notational convention.} As a designer’s understanding of the design space matures, so does the representation that they use. As I already discussed, while the designer is aware of the full expressive powers of formal notations, they only borrow from those notations what they need at the time. However, as the designer progresses and the decisions firm up and additional, less-critical decisions are made, they tend to use more and more of the formal notational convention to represent their commitment to the chosen direction \citep{ossher2010flexible}. The formal elements in Figure \ref{figure:motivation-designbehavior-refinement} illustrate the decisions captured by the designers, where some arrows contain information about cardinality and some elements are contained in a bounding box to reflect their status as an entity in the model. Note that this move towards a more formal notation is not a strictly uniform activity, as different parts of the design may exist at different levels of maturity \citep{petre2009insights}.

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=8cm,keepaspectratio]{./figures/motivation-designbehavior-refinement.png}
  \caption{Diagram from a design session that was refined from a list into a UML diagram.}
  \label{figure:motivation-designbehavior-refinement}
\end{figure*}  

\item \emph{They appropriate a sketch in one notational convention into another notational convention.} Refinement of sketches does not always mean refinement toward and in a single notational convention. Sometimes, designers appropriate one kind of diagram into another \citep{dekel2007notation}. For example, the designers initially created the data model in Figure \ref{figure:motivation-designbehavior-refinement} as a list, then later added boxes, then boxes with arrows, and finally evolved the sketch into a UML class diagram. The designer in all likelihood did not plan this, but in working out their design in place, they re-appropriated the sketch to suit their needs \citep{mangano2012design}. They use what is readily available over re-creating a similar diagram, especially if they do not anticipate needing the original diagram later. 
 \end{enumerate}

\item \emph{They use impromptu notations.} Designers do not exclusively work with the notational convention they know (e.g., UML, ER, etc.), but also, at times, will improvise in the moment. The deviations that they make from standard notations, such as annotating UML diagrams with custom symbols, are deliberate additions that break convention to capture insights before an idea is forgotten. Beyond such annotations and minor deviations, developers also will sometimes adapt wholly new notations on the fly. These often relate to the problem domain that they are explaining, since few domain-specific notations exist, but shorthand is still needed to support the design process \cite{dekel2007notation}. For example, Figure \ref{figure:motivation-designbehavior-impromtu-1} uses freeform text to annotate what looks like a stripped down UML diagram rather than standard UML notation to include additional information. Further, Figure \ref{figure:motivation-designbehavior-impromtu-2} uses rectangular boxes with circles inside to represent a queue of cars waiting for a traffic light, circles with edges to represent traffic lights that govern each queue of cars, and a clock symbol to signify that events in the intersection occur according to a timer component. The designers created these notations on the spot to capture the ideas in their conversation. 

\end{enumerate}

\begin{figure}
  \centering
  \subfigure[UML class diagram] {
      \label{figure:motivation-designbehavior-impromtu-1}
      \includegraphics[width=8.5cm,keepaspectratio]{./figures/motivation-designbehaviors-impromtu}
%      \resizebox{.45\hsize}{.35\hsize}{ }
   }
  \subfigure[Traffic intersection] {
      \label{figure:motivation-designbehavior-impromtu-2}
      \includegraphics[width=6.5cm,keepaspectratio]{./figures/motivation-designbehaviors-impromtu-2}
%      \resizebox{.45\hsize}{.35\hsize}{ } 
   }
   \caption {Diagrams using impromtu notations}
   \label{figure:motivation-designbehavior-impromtu}
\end{figure}

\section{How they use the sketches to navigate through a design problem}
\label{chapter:motivation:navigation}

The second category pertains to how designers use sketches to navigate the design space. While designers may create many different sketches over the course of a design meeting that vary in detail, notation, and what they represent, there is typically a thread of thought that relates the sketches and the ideas represented in them to one another. 

\begin{enumerate}
  \setcounter{enumi}{4}
  \item \emph{They move from one perspective to another.} Software designers create many sketches through which they shift their focus between perspectives. The designer in Figure \ref{figure:motivation-designbehavior-perspectives}, for example, is working on the user interface component for a map, and may navigate to the two different views of the intersection to his left, or explore the UML model of its data model to the far left. The designer uses each new perspective to better understand how the parts of a design fit into the whole, asking questions such as: ``[what] if we look at it like this, from this angle, it fits together like this'' \cite{petre2009insights}. Each perspective presents a new way of looking at the same design, and what may be subtle in one perspective, may be more pronounced and easier to understand in another. 

  \begin{figure*}[tbh]
  \centering
  \includegraphics[width=14cm,keepaspectratio]{./figures/motivation-designbehavior-perspectives.png}
  \caption{The designer navigates between different types of diagrams and abstractions.}
  \label{figure:motivation-designbehavior-perspectives}
\end{figure*}  

 \item \emph{They move from one alternative to another.} In a sufficiently complex software design task, a software designer will generate sketches of competing solutions before committing to a particular choice \cite{zannier2007comparing}. Expressing alternatives as sketches rather than simply mentioning them aloud allows designers to manage their focus and more effectively explore alternative solutions \cite{myers2008designers}. Once created, designers can compare alternatives and weigh their trade-offs \cite{buxton2010sketching} (see behavior 5). They may shift their attention back and forth between alternatives and adopt ideas proposed in one into another, or synthesize the ideas of several alternatives into an entirely new alternative \cite{jones1992design}. For example, the sketches in Figure \ref{figure:motivation-designbehavior-alt} depict two approaches that could be used to model a traffic light, where the left sketch models the light as a single signal with a red, yellow, green, and left turn signal, and the two right sketches use two lights that are separate and make the left turn signal independent from the regular traffic symbol. Placing these diagrams side-by-side helps designers in visualizing and discussing the differences between the two. 

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=5cm,keepaspectratio]{./figures/motivation-designbehaviors-alt}
  \caption{Two diagrams that designers used to discuss alternative approaches for modeling a traffic light.}
  \label{figure:motivation-designbehavior-alt}
\end{figure*}   

 \item \emph{They move from one level of abstraction to another.} Software designers move between different levels of abstraction, either by ``diving into'' parts of their design to explore them in more detail or by shifting ``back up'' to the higher-level representation. For example, the designer in Figure \ref{figure:motivation-designbehavior-perspectives} has navigated to the sketch of a map, which presents a zoomed-out view of the two intersections to his left. This shift in abstraction happens often in software design, as many of its notations are hierarchical in nature. A software architect may shift their focus from working out how software components interact with each other to choosing a component and working out how it functions, perhaps by drawing its internal architecture and diving in even further. This behavior typically leads to a multitude of sketches that together consider different abstractions simultaneously \cite{petre2009insights}. Many scenarios requiring shifts of abstraction have been documented, including the design of user interfaces \cite{da2001user}, web pages \cite{van2003design}, and so on.

 \item \emph{They perform mental simulations.} Software designers use mental walkthroughs to gain insight into the consequences of their design \cite{zannier2007model}. They may need to understand how information flows among components, or inspect their design by mimicking how an end user would interact with it. The software designers `interrogate' their design by testing it against hypothetical inputs and scenarios, often marking over their existing sketch while simulating. For example, while discussing the logic cars use in moving through intersections, the designer in Figure \ref{figure: motivation-designbehaviors-mentalsimulation} runs his finger along the path of the map (shown by the white arrow) to mentally simulate the path that a hypothetical car may take, and verbally walks through the logic the car uses while doing so. Through these mental exercises, the designers can bring to light their implicit assumptions and expose flaws in the design \cite{petre2009insights}. 


 
 \begin{figure*}[tbh]
  \centering
  \includegraphics[width=12cm,keepaspectratio]{./figures/motivation-designbehaviors-mentalsimulation}
  \caption{Designers sometimes use their sketches to mentally simulate their designs in use.}
  \label{figure: motivation-designbehaviors-mentalsimulation}
\end{figure*} 

 \item \emph{They juxtapose sketches.} In order to compare and contrast ideas, software designers will often juxtapose sketches across perspectives, alternatives, and abstractions \cite{petre2009insights}. A class diagram may be examined in parallel with a sequence diagram to aid the designer in determining how a message is passed between components. As another example, the designer in Figure \ref{figure:motivation-designbehavior-juxtapose} points to both a data model and a map to understand how a car object is passed between components as it travels through an intersection. The juxtaposed diagrams help the designer in reasoning how the design might work, using the knowledge gained from one diagram to help identify the omissions or mistakes in another, as well as any inconsistencies between them \cite{petre2009insights}. 

 
\begin{figure*}[tbh]
  \centering
  \includegraphics[width=12cm,keepaspectratio]{./figures/motivation-designbehavior-juxtapose.png}
  \caption{Designers juxtaposing two sketches by pointing.}
  \label{figure:motivation-designbehavior-juxtapose}
\end{figure*} 


 \item \emph{They review their progress.} Not all time spent during design is dedicated towards producing new content or verifying whether the design does what the designer intends it to do. At some point, the designers must take stock of what they have done. They momentarily take a step back, away from the design, and consider the progress that they have made and what they have yet to do \cite{mangano2012design}. They may return to the problem statement or list of requirements and mark off everything they have done to address it, they may generate a new list of issues that they further need to address, or simply just talk amongst themselves, to assess where they are. As an illustration, the designers from Figure \ref{figure:motivation-designbehavior-review} circled a particular subset of their requirements list and explicitly related the items related to that requirement that may have issues that are intertwined with another issue.

 \begin{figure*}[tbh]
  \centering
  \includegraphics[width=12cm,keepaspectratio]{./figures/motivation-designbehavior-review}
  \caption{While reviewing progress, designs may mark up their requirement lists.}
  \label{figure:motivation-designbehavior-review}
\end{figure*} 

 \item \emph{They retreat to previous ideas.} Periodically, designers may reach a stopping point in the exploration of their current set of sketches, such as when they become stuck or simply have exhausted an alternative. They then may choose to return to a previous state of the design (and its sketches) to start anew \cite{zannier2007model}. For example, an abandoned proposal for a time-based architecture may become a more lucrative option if an event-based architecture proves too costly in system memory usage. In returning to past ideas, the designer may bring new insights and a matured understanding from the exploration they just exhausted, which they can use to explore the past ideas further.
 \end{enumerate}

\section{How they collaborate on them}
\label{chapter:motivation:collaboration}

Software design is a highly collaborative activity, especially at the whiteboard where sketching and design exploration is almost always performed in collaboration with others. The behaviors in this section result from the collaborative aspects of working toward a single vision of the design that is shared by all parties.

\begin{enumerate}
  \setcounter{enumi}{11}
  \item \emph{They switch between synchronous and asynchronous work.} While much of the work that takes place at the whiteboard is typically synchronous, with all participants focusing on a single aspect of the design they are discussing, it is known that designers occasionally break away to explore an idea independently while the others continue with the main discussion \cite{dekel2005supporting}. This typically occurs when a sudden inspiration strikes, or when a designer wishes to develop a counterexample or alternative to what is being discussed now. The designers in Figure \ref{figure:motivation-designbehaviors-asynch}, for example, have split to independently work on different aspects of the design. The female on the left is exploring exactly how the timing of an intersection would work using a line graph, whereas the male on the right is working through a UML representation that explores the persistent aspects of the timing mechanism.

 \begin{figure*}[tbh]
  \centering
  \includegraphics[width=12cm,keepaspectratio]{./figures/motivation-designbehaviors-asynch}
  \caption{Designers sometimes break into independent groups to work out solutions, before later synchronizing their insights.}
  \label{figure:motivation-designbehaviors-asynch}
\end{figure*} 

 \item \emph{They explain their sketches to each other.} After any independent work takes place, and even in cases where one designer is drawing on behalf of the group and ``has the floor'', the designers must synchronize their mental models of the state of the design \cite{dekel2007notation}. This behavior relates to Behavior 9, but represents its collaborative version. They will need to verbalize their mental simulations to explain the consequences of a particular choice, clarify the meaning of a sketch, or even simply explain their assumptions or inspiration. While explaining, they may sketch on top of the existing diagram, using their marks to guide attention or add detail, or simply gesture over the diagram if they do not want to edit the sketches.

 \item \emph{They bring their work together.} Sometimes, as a result of asynchronous work, the designers need to integrate their ideas from separate sketches into a unified design. This may involve bringing parts of a sketch over, creating a new sketch that integrates both, or sometimes working on a third alternative that combines the best aspects of each, but requires a different underlying approach to make that work \cite{dekel2007notation}.
\end{enumerate}

\section{Summary}

Taken together, these fourteen design behaviors capture ways in which designers work at the whiteboard, ways that we seek to support in my work. Designers continuously exhibit these behaviors, making choices such as how much or how little they sketch out a design, how they navigate their focus from one sketch to the next, or how they collaborate over these sketches.

These design behaviors, of course, never stand in isolation.  Quite the contrary, any design meeting consists of an interleaved sequence of them.  In explaining a design, a designer may navigate between different perspectives, alternatives or abstractions, and create and refine diagrams of different detail in the process. For example, the designer in Figure \ref{figure:motivation-designbehaviors-interleaving} explains an idea to her design partner (Behavior 12) by mentally simulating a car moving along the map (Behavior 8), and points to different perspectives (Behavior 5) during the mental simulation to show how it affects various parts of the program. 

 \begin{figure*}[tbh]
  \centering
  \includegraphics[width=12cm,keepaspectratio]{./figures/motivation-designbehaviors-interleaving}
  \caption{The designer interleaves many design behaviors.}
  \label{figure:motivation-designbehaviors-interleaving}
\end{figure*} 

Given that these design behaviors occur naturally at the whiteboard, I aim to provide support for all fourteen design behaviors in order to improve the capability of the designer at the whiteboard. The design behaviors must be equally supported, where the support for any particular one should not impede the designer's ability to perform another. As such, I must take an incremental approach in exploring support for these behaviors to ensure any added features meet this criterion.

\chapter{Research Question}
\label{chapter:research-question}

Now that I have introduced this set of design behaviors, I return to what it would mean to build support for it. I first recognize that there is a spectrum of ways in which the design behaviors could be supported. On one end of the spectrum lies the whiteboard itself, a minimally intrusive, informal medium that supports only drawing and erasing of content. The naturally occurring behaviors are permitted, but not necessarily supported, which is the basic problem that this research is trying to address. On the other end of the spectrum is the formal design tool, a highly structured, heavy-weight environment with hosts of explicit features. Examples of such tools are Rational Rose \cite{Quatrani} and ArgoUML \cite{robbins2000cognitive}. Theoretically, they support a number of the behaviors, such as, for instance, Behavior 5 by providing multiple views on the same model, or Behavior 6 by maintaining many projects. They, however, do not nearly support all, and the ones that they do are not nearly as fluidly supported as necessary. 

Between the informal whiteboard and the formal tool lies a range of possible approaches that provide different blends of the strengths of each. This leads to my research question:

    \newenvironment{myindentpar}[1]%
     {\begin{list}{}%
             {\setlength{\leftmargin}{#1}}%
             \item[]%
     }
     {\end{list}}

\begin{myindentpar}{1cm}
\emph{What minimally invasive, coherent set of features can be designed that is sufficient to effectively support these behaviors?}
\end{myindentpar}

I am particularly interested in exploring this question in the context of an electronic whiteboard, that is: what kind of support can we implement that transforms the interactive whiteboard from a place where software designers just sketch like they normally would to a place where their design behaviors are explicitly enabled through the functionality of the tool they are using? Most tools, today, support one behavior or at best a few.

In presenting this research question, it is useful to examine its precise phrasing:

\begin{itemize}

\item \emph{\textbf{minimally invasive} -- I seek to build support that does not completely abandon the whiteboard experience. People go to the whiteboard for its fluidity and flexibility. Any solution that I design must preserve the feel of the traditional whiteboard as much as possible.}

\item \emph{\textbf{coherent set of features} -- I wish to arrive at a set of features that build on each other using a unified set of design principles and metaphors. Building an isolated feature for every behavior would not satisfy my research goal.}

\item \emph{\textbf{sufficient to support all of these behaviors} -- Each behavior should in some way be supported by the overall set of features, and support for any particular behavior should not come at the cost of another. }
\end{itemize}

Having put forward this research challenge, I must analyze the resulting effect of the support for the design behaviors as well. This is a difficult question to answer, particularly since the literature to date has documented that these design behaviors take place, but has not yet fully articulated their effects on the design process and design product. For instance, switching among perspectives (Behavior 5) seems to have a positive effect on the eventual design \cite{baker2010ideas}, and, in certain cases, it has been documented that the consideration of multiple alternatives (Behavior 6) also seems to lead to a better design \cite{buxton2010sketching}. However, even these studies are hard pressed to provide absolute answers. The first study does not examine the optimal length of time a perspective should be explored: is thirty seconds too short, thirty minutes too long, or what is the general distribution? Similarly, the second study does not talk about the number of alternatives: is three sufficient, should twenty be explored, or does it depend on the design problem in some way? These are questions that cannot be objectively answered at this time. 

For this reason, my evaluation must be exploratory. An obvious evaluation would examine factors such as the frequency of design behaviors, time spent working in each behavior, and how my solution impacts the interleaving of the design behaviors. A more comprehensive evaluation would attempt to correlate the appearance of these design behaviors with the quality of the process and the quality of the design by, for instance, relying on outside experts to rate the designs that are produced and correlating this quality with the frequency, interleaving, etc. of the design behaviors. While this would possibly lead to a conclusive statement with respect to the impact of my solution, it would be very challenging to perform this evaluation given that a typical design exhibits complex interleavings of many design behaviors, making achieving any statistical relevance difficult. 

My evaluation, therefore, will take a qualitative approach. By observing Calico in practice and by interviewing users of the tool, I will analyze the usage of Calico within the scope of the design behaviors and analyze the impact of the features on those design behaviors. Additionally, while the observations will likely pertain to the existing set of fourteen behaviors, I of course do not rule out that new behaviors may emerge, as enabled by the features of the software, that might be worthy of study. 

As such, the summary of the approach in the remainder of this dissertation is that I will describe Calico (two different versions) as well as the result of a qualitative evaluation of Calico in use at several local companies.

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 
 \newpage \textsf{•}\chapter{Calico Version One}
\label{chapter:calico-version-one}

Calico Version One is the first complete prototype that was built and evaluated to directly support the design behaviors of Chapter \ref{chapter:motivation}. The prototype was built toward testing the feasibility of my approach. That is, 1) can I successfully build on top of the existing whiteboard experience with features that support the design behaviors, and 2) can I do so without obstructing the design activity itself.

I target only a subset of the design behaviors with the first version of Calico.  In particular, this first versions supports design behaviors 1, 2, 3, and 5, as summarized here:

\begin{itemize}
  \item Design behavior 1: they draw different kinds of diagrams.
  \item Design behavior 2: they produce sketches that draw what they need, and no more.
  \item Design behavior 3: they refine and evolve their sketches over time.
  \item Design behavior 5: they move from one perspective to another.
\end{itemize}

Our express purpose with this first version is to evaluate the viability of the overall idea of supporting design behaviors with tool functionality at the whiteboard.  The features are therefore limited, as compared to the full version of Calico described in Chapter \ref{chapter:calico-version-two}, and the evaluation focuses on building a basic understanding of the effect the features.  Specifically, the evaluation presented in Section \ref{results} looks at how often Calico's advanced features were used, whether use of the features coincided with design behaviors, and whether the resulting design process -- as judged by the nature of the design conversations -- resembles that of design at the regular whiteboard.

In designing the first features of Calico, I avoided those which would inhibit the flexibility and fluidity of the whiteboard. Features from traditional point-and-click interfaces such as different modes, extensive menus, or a selection lasso were not included because such functionality is likely to be frustrating on a pen-based interface \ref{apitz2004crossy} and may disrupt the fluidity of the whiteboard experience. Further, rather than including support by providing sketching support for an existing software design tool (such as with Marama Sketch \cite{Grundy}), I took the approach of building a basic sketching tool first, and adding features incrementally to this foundation.

Through an iterative process of design, experimentation, and careful assessment, I arrived at four features. The features are: (1) a grid to manage multiple canvases in the workspace, (2) scraps that enable advanced manipulation of sketched content, (3) a palette to save and reuse scraps, and (4) a gesture-based interaction scheme to tie all features to stylus-based input.

The remainder of this Chapter is organized as follows. Section \ref{calico} presents Calico and how it addresses the stated goals. Section \ref{experimentaldesign} presents my methods for evaluating Calico. My results are presented in Section \ref{results}. Section \ref{discussion} includes a discussion of the results and Section \ref{threatstovalidity} reviews the threats to validity within the study. Lastly, Section \ref{conclusions} summarizes the chapter and its contributions.

\section{Features}
\label{calico}
In this section, I present the four novel features of Calico Version One, describe them in detail through various examples, and carefully relate them back to the subset of supported design behaviors I target in Chapter \ref{chapter:motivation}. As a guide, Table \ref{table:1} shows a mapping between the subset of four behaviors targeted design behaviors and the main features of Calico.

Before I discuss the features, it should be noted that Calico is intended to be used on an electronic whiteboard (or Tablet PC, in the case of sole person use), and is optimized to take its input from a digital stylus, not a mouse. The designer stands in front of the electronic whiteboard and uses the digital stylus to draw, write, and control Calico (see Figure \ref{fig:2}). The nature of this interaction shaped the design of Calico, particularly the mechanisms with which it enables designers to manipulate a design.

Note that all of the examples I use in this section draw from the evaluation design sessions I describe in Section \ref{experimentaldesign}.

\subsection {Basic Features}
\label{calico:1}

% For one-column wide figures use
\begin{figure}
  \centering
% Use the relevant command to insert your figure file.
% For example, with the graphicx package use
  \resizebox{0.8\hsize}{!}{ \includegraphics{./figures/CalicoVersionOne/figure2.png}}

% figure caption is below the figure
\caption {Physical setup of Calico}
\label{fig:2}       % Give a unique label
\end{figure}
%



\begin{table}
\centering
\caption{Calico features as they address the targeted subset of four design behaviors from Chapter \ref{chapter:motivation}}
\begin{tabular}{ p{4cm}p{2cm}p{9cm} }
\toprule
Design behavior & Feature & Effect \\
\midrule

\multirow{2}{4cm} {Design behavior 1: Use different kinds of diagrams}  &	Scraps	& Scraps maintain the shape of the objects that are drawn, and thus, together with rapid manipulation and creation of arrows, can emulate different notations and diagrams\\ 
	& Palette	 & Scraps can be copied into the palette from which they can later be reused \\
\midrule
\multirow{1}{4cm} {Design behavior 2: Draw only what they need} & Scraps & Scraps do not force the user to draw more than they want to; they require no additional manipulations other than simply drawing the scrap.  The user, thus, can draw as little or as much of a notation as they wish.\\
\midrule
\multirow{2}{4cm} {Design behavior 3: Refine and evolve sketches over time} & Grid & Abstract sketches can be partitioned and evolved across several canvases, which can be meaningfully organized on the grid   \\ 
	& Scraps & Strokes can be transformed from plain sketches to first order objects by making them into scraps, and then related to one another with arrows \\	
\midrule
\multirow{2}{4cm}{Design behavior 5: Navigate between different perspectives} & \multirow{2}{2cm}{Grid} & Sketching effort can be partitioned across multiple canvases \\
	& & Tabs permit quick shifting between, and copying of, canvases \\

\bottomrule
\end{tabular}
\label{table:1}
\end{table}	

Figure \ref{fig:3} presents Calico as it first appears when a developer starts it. Users can immediately draw or write, without needing to enter any mode or selecting a widget to create new content. Just as on a standard whiteboard, they make any marks they wish, anywhere, in any shape. The drawing canvas has just a few visible widgets to maintain the appearance of a standard whiteboard. Seven colored buttons at the bottom left allow a user to change colors. The colors were carefully chosen to be in direct correspondence to the standard pen colors available on a regular whiteboard.

\begin{figure}
  \centering
  \resizebox{0.8\hsize}{!}{ \includegraphics{./figures/CalicoVersionOne/figure3.png}}
  \caption {Calico appearance at the start of a design session}
\label{fig:3}       % Give a unique label
\end{figure}
%

In addition to changing colors, Calico provides a small handful of features familiar to computer users. Users can pan a canvas, save and load designs, export sketches as images for use in other programs, as well as undo and redo actions. The undo and redo actions are globally stored, so if a previous action was performed on another canvas, the user's view will shift to that canvas and the action will be undone. Also, while Calico does not recognize or formalize shapes drawn by the user, Calico does compensate for the typically low polling rate of touch-based hardware by appoximating the curve of the user's strokes using B\'ezier curves. Calico does not support text entry, relying on users to simply write on the board in order to avoid unwanted breaks in concentration due to the need to switch to an extraneous keyboard. While Calico will run on any touch-based device with Windows, Linux, or Mac OS (see Section \ref{calico:5}), all of my experimentation was performed on a board with a projector
 resolution of 1024x768.

\subsection{Grid}
\label{calico:2}

Calico's first novel feature is the grid, which is my answer to the designer's need to not only easily navigate between perspectives (Behavior 5), but to also be able to keep track of their design at hand. Shown in Figure \ref{fig:4}, the grid provides the designer with a bird's eye perspective of the design process as it has unfolded thus far by showing multiple canvases at once. By tapping on a canvas on the grid, the developer enters that particular canvas, where they can create new sketches or modify any sketches already in existence. By tapping the grid icon (see top right of Figure \ref{fig:3}), the designer returns to the grid. 

% For one-column wide figures use
\begin{figure}
% Use the relevant command to insert your figure file.
% For example, with the graphicx package use
  \resizebox{1\hsize}{!}{ \includegraphics{./figures/CalicoVersionOne/figure4.png}}

% figure caption is below the figure
\caption {Grid in use}
\label{fig:4}       % Give a unique label
\end{figure}
%

The grid naturally leads to a partitioning of design effort, while at the same time supporting straightforward movement between the emerging parts of this overall effort. For instance, a designer can make a to-do list in one canvas, use other canvases to work out each item on the to-do list in isolation, and return to the to-do list periodically to check and mark progress. As another example, the designer could choose to examine a design problem from various perspectives or at varying levels of detail in different canvases. The grid becomes the record keeper of this exploration, allowing rapid shifting from one perspective or level of detail to another.

Previous tools have used the filmstrip metaphor to support multiple canvases, with a scrollbar at the bottom, typically in reverse order of manipulation \citep{Stefik}. With frequent updating of the content of different canvases, this leads to a volatile representation in terms of the order of the canvases in the filmstrip, making it difficult to navigate and easily shift between perspectives. The grid advances on the filmstrip by transitioning from a time-based metaphor to a spatial metaphor. The grid keeps all of the canvases in a constant location, which means that a user can not only easily locate individual sketches based on where they exist in the grid relative to each other, but also navigate to adjacent canvases without first having to switch to the grid. They can move left, right, up, or down one canvas at a time by using the tabs that are located in the middle of each edge of the canvas (see Figure \ref{fig:3}). Tapping the white part of the tab enacts the move to the adjacent canvas in that direction.

Tabs also support a different form of navigating between sketches: exploration of alternatives. On a standard whiteboard, it is difficult to fork ideas. One has to manually replicate a sketch elsewhere before modifying it. This is clearly undesirable and either leads to less exploration or direct manipulation of the sketch in question, the latter destroying the original from which the departure is being made and making it difficult to return to a previous state. By tapping the grey half of a tab, a canvas' content is copied in its entirety to the corresponding adjacent canvas, which in turn is placed into focus. With a single ``click'', thus, the designer is provided with a fresh copy of an entire canvas, which they can then further explore, refine, or modify. Using this technique repeatedly, a trail of historical revisions is built that documents how an idea evolved, providing a safety net to always return to previous versions.

Managing different perspectives is further aided by the fact that canvases are moveable on the grid, allowing a user to rearrange the various canvases according to whichever concern they wish to address. That is, they may move ``old'' ideas aside, cluster canvases by phases of design, group topics, or even juxtapose different perspectives.

\subsection {Scraps}
\label{calico:3}

% For one-column wide figures use
\begin{figure}
% Use the relevant command to insert your figure file.
% For example, with the graphicx package use
  \centering
  \resizebox{0.8\hsize}{!}{ \includegraphics{./figures/CalicoVersionOne/figure5.png}}

% figure caption is below the figure
\caption {Scraps}
\label{fig:5}       % Give a unique label
\end{figure}
%

Scraps address a subset three of design behaviors from Section \ref{chapter:motivation:kinds} in Chapter \ref{chapter:motivation}: drawing different kinds of sketches (design behavior 1), creating low-detail sketches that have only as much precision as needed (design behavior 2), and the tendency to move from generic representations to more refined ones (design behavior 3).

Scraps leverage a key action that users of a whiteboard naturally perform: circumscribing some area of sketched or written content \citep{Hendry}. They typically do this to indicate some object of sorts, or to simply mark an area as important. In Calico, the act of circumscription using the secondary button on the digital stylus has visually the same result: the area that is circumscribed is highlighted with a thin border and grey background. In addition, however, the area becomes an object in Calico that can be further manipulated and has several preassigned behaviors. In particular, scraps are implicit groups that are movable, stackable, and relatable. I review these properties one by one.

\emph{Implicit groups}. Scraps build upon the approach taken in Translucent Patches \citep{Kramer}, which allows users to explicitly declare an area as a group. Anything that is either entirely circumscribed in the first place or otherwise written or drawn in this area afterwards is automatically part of the group. Consider the literal sketch of a car visible in the bottom right of Figure \ref{fig:5}. It was first drawn on the canvas, then circumscribed by the stylus to become a scrap. The scrap is now a persistent object with a grey background. Any further additions to the car automatically become part of the scrap.

\emph{Movable}. Scraps are movable. Right-clicking and dragging using the digital stylus will move a scrap and its contents to a different location on the board. This seemingly innocuous action in reality represents a significant improvement over the standard whiteboard: content drawn can be rapidly reorganized. It particularly is important that such reorganization takes place in the language of the user: elements that they have deemed of sufficient importance to promote to being a scrap are the elements that are moved.

\emph{Stackable}. Moving a scrap to a position where some part, or all of it, overlaps another scrap attaches it to the scrap behind it, allowing users to quickly create a stack of scraps (thereby creating hierarchically composed groups), as one would a pile of papers. For instance, the scraps labeled ``Speed'' and ``Position'' in Figure \ref{fig:5} are part of the scrap labeled ``Car Component.'' If ``Car Component'' is moved, ``Speed'' and ``Position'' are moved as well. Dragging a scrap off of another scrap un-groups it. Moving the scrap labeled ``Position'' from its current location on the canvas to where it overlaps the scrap containing the intersection will ungroup it from ``Car Component'' and group it with the intersection scrap in one fluid motion. Note that dragging a scrap implicitly moves it to the top of the order of scraps; scraps do not slide under other scraps. 

\emph{Relatable}. By dragging the digital stylus from one scrap to another, an arrow is created between two scraps. The arrow is persistent and anchored to the places where it originated and ended. When scraps are moved, the arrows move accordingly and keep the two scraps related. In Figure \ref{fig:5}, the scrap ``Position'' relates to the intersection scrap, and the scrap ``Car Component'' relates to the car scrap.

Complementing these basic scrap behaviors are several more specialized behaviors accessible via a small radial menu on the scrap itself. First, a scrap's content can be dropped onto a scrap behind it or, if there is no scrap behind it, back onto the canvas. This allows designers to combine content from multiple scraps. Second, scraps can be given a more regular box shape, and, third, scraps can be made transparent. This latter functionality supports divergence at the micro level of individual scraps (as opposed to the macro level of entire canvases through the grey tabs). By drawing on a translucent scrap that overlays another scrap, developers can explore modifications to this other scrap's content without overwriting its content. If they are satisfied with the result, they could choose to combine the two scraps by dropping the content of the translucent scrap. Alternatives can be compared in this manner as well via multiple translucent scraps capturing different deviations.

Scraps are also fundamental to addressing the behavior of using a mix of notations. By virtue of having physical presence, scraps provide a natural basis for serving as a representation for more structured, though still informally drawn, figures, such as user interface sketches or class diagrams. Since scraps are amorphous, and take on the shape of a designer's stroke, shaping them in visually identifiable forms (e.g., boxes, circles, buttons) allows the designer to informally convey a certain meaning. Particularly when combined with arrows, this supports sketching of numerous types of box-and-arrow-like diagrams. 

\subsection {Palette}
\label{calico:4}

The palette is the third feature that I incorporated in response to the subset of design behaviors I target from Chapter \ref{chapter:motivation}. It specifically addresses the issue of using different types of sketches (design behavior 1).

Palettes are not new, but their typical incarnation in drawing programs is to include a prepopulated set of figures that are not configurable beyond changing the entire set. In design, however, it is not uncommon that a spontaneous notational convention emerges that does not necessarily adhere to any pre-existing or fixed set of figures. Calico's palette, thus, starts empty, and is filled with content by the designer, enabling the reuse of sketched elements. This allows a temporary vocabulary to be created and leveraged within a design session (as exemplified by the impromptu notations in Figure \ref{fig:11}, as further described in Section \ref{results:2}). 

% For one-column wide figures use
%\begin{figure}
%  \centering
%  \resizebox{1\hsize}{!}{ \includegraphics{figure-palette.png}}

% figure caption is below the figure
%\caption {Palette populated with scraps from the lefthand side}
%\label{fig:palette}       % Give a unique label
%\end{figure}
%

The palette leverages scraps for this purpose. Designers can store a scrap simply by dragging it into the palette on the side of Calico's canvas. The palette has a number of cells, each of which may hold one or more scraps. By dragging from a palette cell onto the canvas, any scraps inside that cell are copied to the canvas at the position of the stylus. Once populated, the user can rapidly create variations of a design simply by dragging key scraps from the palette.
Note that the palette serves as a global clipboard; scraps can be stored and reused from any canvas. 

\subsection {Implementation Notes}
\label{calico:5}

Calico Version One's implementation consists of approximately 37,000 lines of code written in Java 6.0. Calico was developed using the Eclipse development environment, and was tested on Hitachi FX-Duo Starboard interactive whiteboards. It is portable across several operating systems (i.e., Windows XP/Vista, Linux, and Mac OSX), though all of my laboratory evaluations described below were performed using a Starboard connected to a Windows laptop that ran Calico. 

\subsubsection {Architecture}
\label{calico:5.1}

Figure \ref{fig:arch} provides a summary of the architecture of Calico, which is based on the model-view-controller pattern. The model is responsible for keeping track of all of the sketched content, including strokes, scraps, arrows, and any relationships that exist among them. Strokes and scraps are stored as sequences of raw coordinates; arrows are stored as a start point and end point. Two relationships are supported. First, when scraps fully contain other content, whether strokes, other scraps, or arrows, a containment relation is kept so that, when scraps are moved, copied, or deleted, the contained elements are also moved, copied, or deleted. Second, an anchoring relation is kept if an arrow's start point or end point is within a scrap, so that when the scrap is moved, the corresponding point also moves. All content is managed by an ObjectHandler, which is responsible to provide not just convenient access, but also accessory methods for creating and restoring from a serialied back-up. 

Lastly, the grid of canvases exists virtually as a set of two-dimensional coordinates within the components in the model. Each stroke, scrap, or arrow has attached an (x, y) identifier to which canvas it belongs. 

The Controller is responsible for interpreting the actions of the user on the canvas and translating those, if needed, to Calico actions. The Gesture Controller is the primary point of access, taking as input the mouse events that are generated when the user interacts with the Hitachi Starboard (I use mouse events only and specifically chose not to take advantage of any special Hitachi Starboard features in order to avoid limiting ourselves to just this electronic whiteboard). Based on the location of the stroke and any pre-existing content that may already exist on its path, the Gesture Controller distinguishes strokes that draw on the canvas from strokes that indicate some action to be performed (see also Section \ref{calico:5}). Strokes that draw on the canvas are passed on directly to the Canvas Controller for creation in the model. Strokes that represent actions are passed on to the Canvas Controller as ``action objects'' using a command pattern, in order to facilitate future extensions with other gestures and actions.

% For one-column wide figures use
\begin{figure}
% Use the relevant command to insert your figure file.
% For example, with the graphicx package use
  \centering
  \resizebox{0.5\hsize}{!}{ \includegraphics{./figures/CalicoVersionOne/arch.png}}

% figure caption is below the figure
\caption {Calico architecture}
\label{fig:arch}       % Give a unique label
\end{figure}
%

The User Interface is built using Piccolo \citep{Bederson}, a zoomable interface that optimizes screen refreshing. Piccolo was chosen because of its available source code and relatively lightweight footprint. The Shape Painters, UI Widgets, Palette, and Grid all extend the Piccolo Framework and comprise the interface with which the user interacts. The Shape Painters mirror the data model using nodes from the Piccolo Framework to draw content on the canvas; any changes to the data model are then automatically propagated. The UI Widgets take care of Calico functionality such as the tabs, panning, and undo and redo. The Palette also exists as a separate UI widget, albeit with special functionality to hold scraps that can be reused. Upon initialization, Calico instanciates a fixed number of canvases and stores them in the grid, which can be referenced using the (x,y) identifier. 

\subsubsection {Gesture-Based Input}
\label{calico:5}

Permeating across the subset of four targeted design behaviors from Chapter \ref{chapter:motivation} is the need for gesture-based input. Many of the creative, exploratory activities that Calico supports rely on the fluidity and quickness of sketching. Interactions supplemental to the primary sketching activity should be equally quick. The design flow must be maintained, shapes should be created the way designers want them, and the effort of making changes should be low. Essentially, ``viscosity'' \citep{petre2009insights}, the cost of making changes, has to be low or designers will be discouraged from exploring opportunities in design. 

In order to maintain the natural feel of sketching in the interaction, I have chosen to use a mixed mode approach: many features are ready-at-hand through simple gestures, but some more esoteric functionality that would otherwise require integration of complicated gestures is available in a pie menu off of scraps (visible in Figure \ref{fig:gesturespie}).

\label{results:22}

\begin{figure}%
  \centering
  \subfigure[Create scrap] {
      \label{fig:gesturesa}
      \resizebox{.2\hsize}{.15\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure-gesturesa.png}}
   }
  \subfigure[Delete scrap] {
      \label{fig:gesturesb}
      \resizebox{.2\hsize}{.15\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure-gesturesb.png}}
   }
  \subfigure[Expand scrap] {
      \label{fig:gesturesc}
      \resizebox{.2\hsize}{.15\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure-gesturesc.png}}
   }
   \subfigure[Pie Menu] {
      \label{fig:gesturespie}
      \resizebox{.2\hsize}{.11\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure-gestures-piemenu.png}}
   }



  \subfigure[Create arrow] {
      \label{fig:gesturesd}
      \resizebox{.4\hsize}{.15\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure-gesturesd.png}}
   }
  \subfigure[Delete arrow] {
      \label{fig:gesturese}
      \resizebox{.4\hsize}{.15\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure-gesturese.png}}
   }
  \subfigure[Expand arrow] {
      \label{fig:gesturesf}
      \resizebox{.4\hsize}{.15\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure-gesturesf.png}}
   }
  \subfigure[Contract arrow] {
      \label{fig:gesturesg}
      \resizebox{.4\hsize}{.15\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure-gesturesg.png}}
   }
   \caption {Gestures in Calico for scraps and arrows}
   \label{fig:gestures}
\end{figure}%

Interaction is driven by a small set of context-sensitive gestures, as shown in Figure \ref{fig:gestures}. Each stroke of the stylus performs a unique action depending on where it travels. For example, dragging the pen while holding the secondary stylus button on the canvas creates a scrap, but performing the same action on a scrap will move the scrap. With ordinary strokes, dragging the stylus along the canvas draws directly onto the canvas, but slashing it across a scrap, as shown in Figure \ref{fig:gesturesb}, will delete it. The gestures take into consideration where the stylus begins, which object(s) it intersects along its path, and where the stylus ends. This is sufficient to build a straightforward interaction scheme that allows the user to create scraps, manipulate them in various ways, draw arrows from one scrap to another, and navigate the entire Calico interface with ease. Each action has a priority, such that stylus strokes spanning multiple interactions behave predictably. For instance, when a user draws a line from one scrap to another that crosses an existing arrow, a new arrow is created and the existing arrow is not deleted, despite the fact that the stroke struck through it (a gesture that normally deletes that arrow).

\section{Experimental Design}
\label{experimentaldesign}

To evaluate Calico Version One, I conducted an exploratory, comparative study in which I asked pairs of participants to perform a software design task using either Calico or a regular whiteboard. My goals were threefold: first, I wanted to ensure that, when Calico is used for an actual design activity, people moved beyond basic whiteboard sketching and used Calico's advanced features. Second, given that Calico was designed to support specific design behaviors, I wished to observe whether these behaviors actually did occurred. Lastly, I wished to assess the structure of the design conversation in the pairs that used Calico, and compare it with that of the pairs that used the whiteboard.

\subsection {Recruitment and Participants}
\label{experimentaldesign:1}

Sixteen pairs of participants (32 participants in total) were given one hour and fifty minutes to complete a given design task. Eight of the pairs used Calico to perform the design task, and eight of the pairs used the regular whiteboard. 
I recruited participants at my university using email advertisements and snowball sampling. In a pre-experiment survey, I asked participants to declare their area of expertise, and their industrial experience. All participants were computer science graduate students with some degree of experience designing software systems. The average amount of industrial experience was three years, and ranged from none to seven years of experience. To balance pairs, I paired participants with the similar industrial experience together to avoid having the more experienced designer take over the session. I then gave each pair a rating based on the average experience of its members, and distributed these pairs equally among the whiteboard and Calico conditions so that each condition had an equal amount of experienced and inexperienced pairs.

\subsection {Procedure}
\label{experimentaldesign:2}

Each session lasted between two and two a half hours. When participants arrived, they were given an informed-consent form that included details about the nature of the study. Those in the Calico session were given a 30-minute tutorial, while those in the whiteboard activity were allowed to begin immediately, upon which they were given the two page prompt (see below). Participants were asked to use the whiteboard and to not write on the prompt or any other paper so that cameras could have a clear view of anything they wrote. Pairs were given one hour and fifty minutes of design time, after which they briefly recapped their design in their own words (5-10 minutes). After pairs finished their design activity, they were given five minutes to collect their thoughts and then to summarize their design in a ten-minute explanation. Afterward, I interviewed them for ten minutes about their opinions and experience concerning the activity, and their past experience with similar technology. At the end of the session, each participant was given \$100 as an incentive and a \$250 prize was awarded to the pair with the best design. As a result, all pairs took the exercise seriously and were fully engaged throughout the design exercise. 

Each session was recorded on video camera for subsequent analysis. Additionally, Calico produced detailed logs that captured each individual user action (e.g., scribble drawn; scrap created, moved, or deleted; switch canvas in the grid).

\subsection {Task}
\label{experimentaldesign:3}

Each pair received the same design prompt, asking for the design of an educational traffic signal simulator to be used by students in a civil engineering course (the same prompt was used in the Studying Professional Software Design workshop; it is included in its entirety in the introduction to the Design Studies journal special issue dedicated to this workshop \citep{Petreb}). The prompt, provided a series of open-ended goals and requirements, asking the pairs to design a system that allowed engineering students to: (1) create a visual map of the roads, (2) describe the behavior of the lights at each intersection, (3) simulate traffic flow, and (4) change parameters of the simulation, such as traffic density. The prompt also instructed pairs to produce a design that they could present ``to a pair of software developers who will be tasked with implementing it.'' 

\subsection {Measures}
\label{experimentaldesign:4}

Our goal was to: (1) measure how often Calico's advanced features were used, (2) examine how those features were used with respect to the design behaviors I outlined earlier, and (3) assess the structure of the design conversation in both the whiteboard and Calico sessions. I also asked a standard set of exit interview questions concerning the group's satisfaction with the tool and the design process that they followed.

    \emph{Use of features}. In order to objectively verify that participants move beyond just sketching like they would on a regular whiteboard and actually use the advanced features of Calico, I measured the amount of usage that each feature received. To perform this, I reviewed the videos and made a note of each time a particular feature was used. 

    \emph{Design behaviors}. After I had recorded all instances of the features being used, I performed a qualitative analysis of how the features were used. I was interested in seeing how well those features supported the targeted subset of four design behaviors from Chapter \ref{chapter:motivation}. For the first design behavior, using different types of diagrams, I watched out for situations where participants mixed several notations together in a single canvas, and highlighted occurrences of impromptu notations that were unique to a particular pair and did not appear in others. For the second design behavior, drawing only what they need, I noted the general diagrams that participants sketched, and paid particular attention to what models participants created, and if and how they used scraps to do so. For the fourth design behavior of moving from abstract to concrete, I compared photos of diagrams at various stages, and noted how pairs restructured their diagrams and added additional detail. When considering the fifth design behavior, navigating between perspectives, I was particularly attentive to the reason that participants moved out of one canvas and into another within the grid, and I classified the general activities that occurred in each of the canvases as well.

    \emph{Structure of Design Conversations}. After qualitatively assessing the videos for evidence of the design behaviors, I assessed the impact that Calico had on the structure of the design conversation by coding audio transcripts of the design sessions for what kinds of activity took place at each moment. The categories and guidelines for the coding scheme were adopted, with minor changes, from a previous study on design meetings \citep{Olson}. As the authors of that study explain, the categories were derived from the Design Rationale literature \citep{Moran} as well as studies of group activity \citep{PUTNAM}, and reflect key aspects of the design activity. With respect to the activities from design rationale, statements were separated into \emph{Issues} at hand, the \emph{Alternatives} or solutions raised, and the \emph{Criteria} used to evaluate an idea. Within the same coding scheme, statements could also be organized into organizational activities, i.e., conversations the group had to organize itself (specifically called \emph{Meeting Management}, \emph{Summary}, \emph{Walkthrough}, and \emph{Goal}), to \emph{Clarify} their ideas, or to engage in \emph{Digressions}. Two additional categories not part of the original coding scheme, \emph{Technology Management} and \emph{Technology Confusion}, were inspired by a previous study based on the same coding method that analyzed a tool's effect on the design process \citep{Olsonb}. \emph{Technology Management} refers to the times when the participants' focus was devoted to the tool itself rather than the activity at hand, and \emph{Technology Confusion} refers to time lost due to system failure. Lastly, any category that did not fit into any of the above was categorized as \emph{Other}. 

Due to limited time and resources, only a subset of the sessions, specifically six Calico and six whiteboard sessions, were coded (see Table \ref{table:3} and Table \ref{table:4}). While this is not enough to claim statistical significance, it is enough to gain a sense if there is, in fact, an apparent difference in process between the two conditions.

In order to verify the validity of the coding, I performed an interrater reliability test, and also consulted with researchers who previously applied this coding in their own past studies. After initial training, two individuals independently coded a session, and then compared their coded transcripts to perform an interrater reliability test. I obtained a Cohen's k value of 82\% at this level of granularity. I then compared the total time that each coder had for each category, and found a correlation of .998. These measures are well within the accepted tolerance for behavioral analysis.

    \emph{Satisfaction and perceptions of participants}. At the end of each session, I interviewed participants in order to learn how Calico affected their approach to the design task, and what the positive or negative aspects of Calico were for the users. I did this by first asking participants to reflect on their designs and the process they used to get there. I then asked them explain how they used each of the advanced features, their overall satisfaction with each feature, and whether they had any suggestions for improvement. 

\section{Results}
\label{results}



% For one-column wide figures use
\begin{figure}
  \centering
% Use the relevant command to insert your figure file.
% For example, with the graphicx package use
  \subfigure[] {
  	\label{fig:6a} 
  	\resizebox{0.64\hsize}{0.36\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure6a.pdf}} 
  }

  \subfigure[] {\label{fig:6b} \resizebox{0.64\hsize}{0.36\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure6b.pdf}} }

  \subfigure[] {\label{fig:6c} \resizebox{0.64\hsize}{0.36\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure6c.pdf}} }

% figure caption is below the figure
\caption{Feature usage across the eight calico pairs}
\label{fig:6}       % Give a unique label
\end{figure}
%


The discussion of my results is organized by the major categories of analysis described in the previous section: use of features, design behaviors, design conversations, and satisfaction. In this section, I only present the results that I observed; I do not make attempts to interpret them. In Section \ref{results:2}, I bring the results together and draw my conclusions about the value of Calico.

\subsection{Feature use}
\label{results:1}

I first focused on whether participants would move beyond just basic sketching to using the advanced features of Calico. This point is important, given that there was no incentive for participants to use any of the advanced features other than the features being useful to their task at hand; they could have instead chosen to sketch as they normally would on a standard whiteboard without the help of scraps or the grid. 

Figure \ref{fig:6} shows the result, marking each time the grid (a), scraps (b), or palette (c) was used. From the graph, it can be seen that the grid was unanimously used by all pairs, with heavy usage by most pairs and moderate by some. The pairs predominantly switched to the grid view first for navigating to different canvases, though as time went on, several incorporated the use of the tabs to navigate to adjacent canvases (see Section \ref{results:21}). 

There was a wide distribution in the frequency of scrap use, with some pairs strongly relying on scraps, others exhibiting more moderate use, and a few barely using them. The palette was the least used feature, with just two pairs using it some and three other pairs using it twice each.

\subsection{Design behaviors}
\label{results:2}

Given my goal of supporting the specific subset of four design behaviors of the total set in Chapter \ref{chapter:motivation}, this section focuses on occurrences of these behaviors in the various pairs, as well as how the advanced functionality of Calico was used in the presence of these behaviors.

\subsubsection{Design behavior 1: Drawing different types of diagrams}
\label{results:23}

As expected, numerous notations were used by the participants as part of their design process. These include class diagrams, user interface components, and diagrams specific to the domain of traffic simulation. These different types of diagrams were often created in response to one another. In the whiteboard sessions, this led to heterogenous content in different notations spread out over the whiteboard. In the Calico sessions, the pairs broke up their designs across different canvases, as exemplified in Figure \ref{fig:7}, and as a result most canvases used just a single notation. Pairs sometimes did mix different notations in a single canvas, such as in the Figure \ref{fig:8d}, where one of the pairs uses representations of a traffic intersection in the top of the image with pieces of code representations in the bottom-right of the image, on the same canvas. This pair, as well as other pairs that similarly mixed representations, used the different representations to juxtapose different views of the design.

Pairs had no ready notation to represent traffic structions, and so they created their own on the fly. For instance, the image in Figure \ref{fig:9-a} shows how one pair used scraps to model a state diagram that is part of their vision for how civic engineering students will specify the timing of traffic lights in their simulation. Note how the state diagram exists on one of two tabs meant to be part of the user interface. Next, in Figure \ref{fig:9-b}, the same pair applied a similar notation to define the logic that is executed once a car arrives at an intersection. Here the pair reused the same diagrammatic elements by copying scraps. In Figure \ref{fig:9-c}, another pair created dozens of small scraps and placed them in a grid configuration to simulate the interface for traffic flow. They filled in select canvases with the pen to simulate a particular route, and experimented with different routes by erasing and filling in other scraps. In all three examples within Figure \ref{fig:9}, pairs were able to create copies of their scraps and reuse them to attempt alternate combinations. 

\begin{figure}%
  \centering
  \subfigure[State diagram for defining light combinations at an intersection] {
     \label{fig:9-a}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure9a.png}}
   }
  \subfigure[State machine that is executed when a car arrives at an intersection] {
      \label{fig:9-b}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure9b.png}}
   }
  \subfigure[Simulation of traffic flow on the map and the controls to regulate it] {
      \label{fig:9-c}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure9c.png}}
   }
   \caption {Several impromptu notations emerged in the design sessions}
   \label{fig:9}
\end{figure}%

\subsubsection{Design behavior 2: Drawing only what they need}
\label{results:22}

\begin{figure}%
  \centering
  \subfigure[] {
      \label{fig:8a}
      \resizebox{.45\hsize}{.35\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure8a.png}}
   }
  \subfigure[] {
      \label{fig:8b}
      \resizebox{.45\hsize}{.35\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure8b.png}}
   }
  \subfigure[] {
      \label{fig:8c}
      \resizebox{.45\hsize}{.35\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure8c.png}}
   }
  \subfigure[] {
      \label{fig:8d}
      \resizebox{.45\hsize}{.35\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure8d.png}}
   }
   \caption {Examples of models in varying amounts of detail}
   \label{fig:8}
\end{figure}%

I observed the use of low-detail models in every single design session. The pairs that used Calico created the same type of low-detail models that I saw on the whiteboard, and in most cases they used scraps to create these models. Of the eight Calico pairs, two simply sketched directly on the canvas as they would have on a traditional whiteboard, and did not further manipulate their sketches. In the cases where pairs did use scraps, they benefited from the extra functionality, such as moving scraps to reorganize a design, copying them in order to create variations, and arrows to define relatioships.

The images in Figure \ref{fig:8a} and \ref{fig:8b} illustrate an example of how many of the pairs used scraps to create box-and-arrow representations of code structures. The pairs that did not use scraps, such as the pair that produced Figure \ref{fig:8c}, drew boxes and arrows, but did not heavily restructure their diagrams. The pairs that did use scraps engaged in more organization of their box-and-arrow diagrams through the moving and copying of scraps. Additionally, they typically refined their scrap-based models into UML-like models, showing how scraps can be used as the basis for such refinement from low detail to more detail. In the case of Figure \ref{fig:8b}, the designers annotated some of their scraps with ``I'' to indicate that it is an interface, and on the right-hand side they created lists of parameters for each object.

Many of the other diagrams produced in Calico shared similar qualities with the low-detail diagrams, with Figures \ref{fig:8c} and \ref{fig:8d} providing two more examples. These diagrams were quick sketches that participants initially used to help them understand a particular situation. The designers in Figure \ref{fig:8c} did not use scraps heavily and so they experimented with traffic configurations by simply drawing and erasing different scenarios. The designers in Figure \ref{fig:8d} used scraps to accomplish a similar task. In this case, they used scraps of different shades to represent traffic configurations, and experimented with different configurations by moving and copying the scraps. Additionally, they used scraps to pull pieces of the design from surrounding canvases in order to understand how a particular traffic configuration would work within the context of other parts of the system. 



\subsubsection {Design behavior 3: Refinement of representations}
\label{results:24}

Both the grid and scraps were used in the refinement of representations over time. At the most general level, the grid partitioned the design space so that participants could separate high level and low level representations. Many pairs used spatial orientation to navigate from higher level representations to lower level representations, where higher level representations would exist at the typically left-most canvases, and the details of components in adjacent canvases to the right or below the originating canvas. Next, when members made the transition to more concrete representations, they would transform sketches into scraps, and create copies on other canvases where they would expand them in more detail and create relationships between them. 

There were two patterns of this behavior that occurred frequently within pairs. Figure \ref{fig:11} illustrates a representative example of the first pattern, which occurred in over half of the Calico pairs. Most pairs, but not all, began by creating several lists of design requirements, goals, and what they viewed as major aspects of the system. The example in Figure \ref{fig:11-a} contains several high-level components such as Maps, Intersections, Cars, and so on. After brainstorming lists such as these, pairs would commonly convert them into scraps (Figure \ref{fig:11-b}) and copy them into another canvas, where they would expand on these in more detail, as in Figure \ref{fig:11-c}. However, creating lists such as these was not common in all pairs, two pairs chose to jump right to diagramming. In the exit interviews, these two pairs both reported that they were first concerned with understanding the requirements and the world that they were modeling. They both began by creating concrete representations of what they knew was true within intersections, and then developed a high level understanding by drawing on top of and discussing these models with their partner. To accomplish this, they created many diagrams of intersections, and from these jumped to writing down assumptions in lists contained in other canvases. When they encountered a new concept that they sensed was complex, such as how to handle the timing within intersections, they would loudly say, ``let's leave this for later'', and leave it as a generic annotation.

A second pattern that occurred more frequently was the breaking down of the design across multiple canvases. A representative example is depicted in Figure \ref{fig:12}, in which the participants from that pair partitioned their code structure across many spaces. They created a high level perspective of the architecture in one canvas using scraps (Figure \ref{fig:12-a}) and then copied these scraps to adjacent canvases, where a particular scrap would be expanded to include more detail. In Figure \ref{fig:12}, the participants divided the architecture of their program into UI, Map, and Simulation Logic. They copied the contents of this canvas to an adjacent canvas, where they subsequently fleshed out the details of Simulation Logic, but left Map and UI with no additional detail. They then used a third canvas to work out the details of Map, while leaving Simulation Logic and UI as is. Using this divide and conquer strategy, the participants were able to effectively partition their design. This behavior of spreading diagrams across several canvases is similar to what Dekel and Herbsleb \citep{dekel2007notation} observed in their studies, where diagrams in one drawing space would depend on references located in other spaces. 

\begin{figure}%
  \centering
  \subfigure[List of high level objects] {
     \label{fig:11-a}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure11a.png}}
   }
  \subfigure[List of high level objects converted to scraps] {
      \label{fig:11-b}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure11b.png}}
   }
  \subfigure[High level objects refactored and expanded] {
      \label{fig:11-c}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure11c.png}}
   }
   \caption {Lists were converted into representations of software components using scraps}
   \label{fig:11}
\end{figure}%

\begin{figure}%
  \centering
  \subfigure[High level perspective of system architecture] {
     \label{fig:12-a}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure12a.png}}
   }
  \subfigure[Definition of Simulation Logic] {
      \label{fig:12-b}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure12b.png}}
   }
  \subfigure[Definition of Map] {
      \label{fig:12-c}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure12c.png}}
   }
   \caption {Representations of software components were broken up across several canvases}
   \label{fig:12}
\end{figure}%

\subsubsection {Design behavior 5: Navigating between perspectives}
\label{results:21}

\begin{figure}%
  \centering
  \subfigure[List of requirements in bullet point form] {
     \label{fig:7-a}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure7a.png}}
   }
  \subfigure[UI mockup of the traffic simulator interface] {
      \label{fig:7-b}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure7b.png}}
   }
  \subfigure[Architecture in the form of boxes and arrows] {
      \label{fig:7-c}
      \resizebox{.3\hsize}{.4\hsize}{ \includegraphics{./figures/CalicoVersionOne/figure7c.png}}
   }
   \caption {Participants tended to focus their efforts towards a particular theme in each canvas}
   \label{fig:7}
\end{figure}%

As the heavy use of the grid would suggest, Calico assisted designers readily in shifting focus. All sessions resulted in grids similar to the one shown in Figure \ref{fig:4}, with the designers moving back and forth among canvases frequently. Pairs switched to a new canvas to either: (1) address additional detail generated by the current sketch, or (2) generate a new alternative. In the majority of cases it did not matter where on the grid a pair went to continue their design activities, as a consequence of which I observed quite a few linear chains of canvases on the grid in use. Several interesting cases did emerge, however. For example, in one session, each of the pair members ``owned'' a grid row of design content in that they were the primary designer for all of the sketches in their row. As another example, one pair used the spatial metaphor extensively, with members talking about moving in a given direction (i.e., left, right, up, down) to navigate to certain sketches. While not all pairs operated as explicitly in terms of direction, the grid's spatial orientation did provide a consistent layout that enabled the pairs to shift focus by navigating to different canvases that they knew existed in certain locations.   

A number of pairs exhibited bursts of back-and-forth switching between canvases to compare content, as illustrated in Figure \ref{fig:6a} by the overlapping sequences of dots. This happened for two reasons. First, participants moved back-and-forth when they needed to mentally juxtapose two concepts, such as when they were working to improve their design for the user interface in conjunction with the underlying model. Second, they would take stock to verify that the different sketches were consistent with one another to make sure they had not inadvertently introduced problems when they furthered some aspect of their design.

The contents of the canvases tended to break down in three categories: (1) requirements, (2) code structures, and (3) UI mockups, with examples of each depicted in Figure \ref{fig:7}. Every pair used the first canvases to lay out their requirements in a list, as in Figure \ref{fig:7-a}, after which they would move to another canvas to embark on the design proper, producing diagrams similar to Figures \ref{fig:7-b} and \ref{fig:7-c}. Pairs would frequently return to the requirements for inspection and assessment of their progress.

The ability to copy canvases proved useful not only to generate alternatives (one can see several variant sketches in Figure \ref{fig:4}), but also when it was desirable to divide the contents of a canvas between two canvases so to be able to work on each part separately. Copy followed by subsequent deletion of the respective halves of the copied content achieved this desired result. One of the pairs used the copy feature to create backups of individual canvases, and moved these backup copies out of the way by dragging the canvases to the outer edges of the grid. Several other pairs moved canvases in the grid around to more clearly organize their design.

Overall, I observed that participants use the grid as a tool to divide design content across canvases, where a shift in attention was commonly accompanied by a change of canvas, and long periods of attention were marked by extended periods of continued activity in a single canvas (visible as gaps between points in Figure \ref{fig:6}). Additionally, I observed the spatial layout to be helpful to designers, both in providing clear references to design content and in organizing the overall design effort.


\subsection {Structure of Design Conversations}
\label{results:3}

I now examine the structure of the design conversations within the Calico pairs, and see how it compares with the pairs that performed the activity on the whiteboard. 

\subsubsection {How time was spent}
\label{results:31}

I first determined the structure of the design process by measuring the total time spent in each design category during conversation. I coded the transcripts of the spoken parts of the meetings, summarizing the total time spent in each category, as shown in Table \ref{table:2}. 

\begin{table}
\centering
\caption{Summary of design conversation categories.}
\begin{tabular}{ p{3.5cm}p{1.5cm}p{1.5cm}p{1.5cm}p{1.5cm} }
\toprule
Category & Whiteboard Pairs & Sum & Calico Pairs & Sum \\
\midrule
Issue 		& 6.37		&	& 7.34 		&  \\
   Clar Issue	& 1.06		&	& 0.15 		&  \\
Alternative	& 37.04	&	& 41.83 	& \\
 Clar Alt	& 2.08		&	& 1.30 		&	\\
Criteria		& 15.63	& 	& 16.47 	&	\\
 Clar Criteria	& 0.48		&	& 0.42		& \\
		&		& 62.66 &		& 67.50 \\
General Clar	& 0.04		&	& 0.08		&	\\
Artifact Clar	& 0.11		&	& 0.21		&	\\
Summary	& 1.32		& 	& 1.57		&	\\
 Clar Summary & 0.00	&	& 0.41		&	\\
Walkthrough	& 2.05		&	& 1.90		&	\\
 Clar Walk	& 0.00		&	& 0.07		&	\\
Goal		& 1.07		&	& 0.91		&	\\
 Clar Goal	& 0.52		& 	& 0.18		&	\\
Meeting Mgmt	& 6.37		& 	& 7.34		&	\\
 Clar Meeting Mgmt & 0.00 &	& 0.01		&	 \\
Digression	& 0.82		&	& 0.48		& 	\\
 Clar Digression & 0.00	&	& 0.00		&	\\
Manage Technology & 0.35	&	& 3.66		&	\\
Technology Conf & 0.04	&	& 5.09		&	\\
Other		& 0.00		&	& 0.91		&	\\
		&		& 12.43 & 	& 22.79 \\
Pause		& 24.01	& 24.01 & 9.67 & .67 \\
\bottomrule
\end{tabular}
\label{table:2}
\end{table}	

Categories related to design rationale (i.e., \emph{issues}, \emph{alternatives}, and \emph{criteria}) took up the majority of the design sessions. In the Calico pairs, these took up 67.5\% of the meeting time, and in the whiteboard pairs they took up 62.7\% of the meeting. Both Calico and the whiteboard had a similar breakdown within the design rationale categories. Within the Calico and whiteboard pairs, discussing \emph{alternatives} occupied 41.8\% and 37.0\% of the time, respectively. Both pairs spent roughly the same amount of time discussing \emph{issues}, at 7.3\% and 6.4\% for Calico and whiteboard pairs, respectively, and also the same amount of time evaluating solutions in the \emph{criteria} category, with 16.5\% and 15.6\% for Calico and whiteboard, respectively. However, the average length of stay in each category, i.e. the average uninterrupted time for a particular category, was higher for the Calico pairs than the whiteboard pairs. The average length of stay for \emph{alternative} in Calico pairs was 13.0 seconds compared to 8.7 seconds for whiteboards. The average length of stay for \emph{criteria} in Calico pairs, 11.0 seconds, was also longer than that for the whiteboard pairs, 7.5 seconds. However, the average length of time per \emph{issue} proposed was similar, with Calico pairs averaging 8.1 seconds and whiteboard pairs average 7.1 seconds.

Roughly double the amount of time was devoted to management related activities in Calico pairs compared to whiteboard pairs, however this time difference was due to discussion of the technology. Management related categories occupied 22.8\% of the meeting in Calico, while they only occupied 12.4\% of the meeting in the whiteboard pairs. Of that time in Calico, 8.8\% was spent discussing the technology (combined sum of \emph{Management of Technology} and \emph{Technology Confusion} categories). Both pairs spent relatively the same amount of time explicitly discussing meeting management, 7.3\% and 6.4\% for Calico and whiteboard pairs, respectively. The other categories were roughly similar as well.

I also recorded the amount of time spent not talking, which I recorded as \emph{pause}. I found that the whiteboard pairs were quiet for more than double the period of Calico pairs, with whiteboard pairs silent for 24.0\% compared to 9.7\% for Calico pairs. Upon closer inspection, I saw that the majority of whiteboard pairs used this period of silence to work independently of each other. Calico pairs were not given this opportunity since the system only permits one user to write at a time. While I do not measure the impact that this had here, during the exit interview participants reported frustration over their inability to work independently. As a result, they reported that they had to force their attention on what the other person was writing, and abandon ideas they were thinking of independently because they could not write them down.

I then recorded time spent clarifying answers across all categories, though I found that little time was dedicated to clarification within both sessions. The combined percentage of clarification across all categories for the Calico pairs was 2.8\%, while it was 4.3\% for the whiteboard pairs. 

Lastly, I examined pairwise levels of similarity across all pairs, shown in Table \ref{table:3}. I compared the average time spent within each category across Calico and the whiteboard conditions, and found a correlation of .92, indicating a strong relationship. All Calico pairs had a strong correlation with one another, ranging from .90 to .98. Within the whiteboard pairs, there was a much larger variability, with correlations ranging from .16 to .98, with a median of .58. A major way in which the meetings varied, particularly between Whiteboard Session 2 and Session 5, was in the amount of time that was categorized as \emph{pause}. Removing the pause category from the pairs raises the correlations significantly to the range of .93 to .99 (not visible in table). Overall there appears to be much similarly across all sessions, with pairs within the Calico condition having the tightest range when considering all of the categories.

\begin{table}
\centering
\caption{Correlations between sessions for total time spent (dark cells pertain only to Calico sessions, white cells pertain only to whiteboard sessions, and lightly shaded cells are the intersection of both)}
\begin{tabular}{ p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}}
\toprule
&	C.1 &	C.2 &	C.3 &	C.4 &	C.5 &	C.6 &	W.1 &	W.2 &	W.3 &	W.4 &	W.5 \\
\midrule
C.1 	 & & & & & & & & & & & 								 \\		
C.2  &	0.90 & & & & & & & & & & 								 \\		
C.3 & 	0.98 &	0.94 & & & & & & & & & 							 \\		
C.4 & 	0.97 &	0.96 &	0.98 & & & & & & & & 						 \\		
C.5 & 	0.94 &	0.98 &	0.97 &	0.98 & & & & & & & 						 \\	
C.6 & 	0.95 &	0.95 &	0.98 &	0.98 &	0.97 & & & & & & 					 \\	
W.1 &	0.94 &	0.96 &	0.97 &	0.96 &	0.99 &	0.97 & & & & & 				 \\	
W.2 &	0.41 &	0.20 &	0.44 &	0.33 &	0.27 &	0.43 &	0.29 & & & & 			 \\	
W.3 &	0.92 &	0.97 &	0.94 &	0.95 &	0.98 &	0.94 &	0.98 &	0.20 & & & 			 \\
W.4 &	0.92 &	0.98 &	0.94 &	0.98 &	0.99 &	0.95 &	0.98 &	0.16 &	0.99 & & 		 \\
W.5 &	0.61 &	0.55 &	0.73 &	0.67 &	0.61 &	0.74 &	0.61 &	0.92 &	0.54 &	0.53 & 	 \\
W.6 &	0.91 &	0.91 &	0.95 &	0.93 &	0.93 &	0.97 &	0.93 &	0.57 &	0.91 &	0.90 &	0.83 	\\
\bottomrule
\end{tabular}
\label{table:3}
\end{table}

\subsubsection {Transitions Between Activities}
\label{results:32}

\begin{figure}
  \resizebox{1\hsize}{!}{ \includegraphics{./figures/CalicoVersionOne/figure13.png}}
\caption{How time was spent within and between the design activities}
\label{fig:13}      
\end{figure}
%

In addition to total time, I also examined all incoming and outgoing transitions that pairs made between categories in order to understand the flow of design conversations. I first computed the total number of times the participants transitioned from one category to another, which is shown in Figure \ref{fig:13} through arrows, the thickness of which representing the relative frequency that a particular transition occurred. The most common transitions that occurred belonged to the design rationale categories: \emph{issues}, \emph{alternatives}, and their \emph{criteria}. In Calico they were involved in 76.8\% of all transitions, and 77.7\% of all transitions in the whiteboard pairs. 

I then computed the correlation matrix for the Calico and whiteboard pairs, and found striking similarities between the transitions of all 22 categories. In order to get the correlation matrix, I computed the average percentage that a given transition happens between one category to the next, and established the correlation matrix shown in Table \ref{table:4}. The average transition that happens in the Calico pairs was strongly correlated with that of the whiteboard pairs at .92. As with the previous table of correlations, the Calico pairs, on average, are more highly correlated with one another than the whiteboard pairs and have a tighter range, The Calico pairs range from .63 to .93, compared to whiteboard pairs, which range from .41 to .98. As with the previous table of total time spent, whiteboard pairs 2 and 5 also have low correlations with the other whiteboard pairs, and a high correlation with each other, suggesting a distinctive difference in work style. Also, unlike the previous table, Calico pair 4 correlates slightly less, on average, than other Calico pairs, and much less with whiteboard pairs, suggesting they used a slightly different process as well.

\begin{table}
\centering
\caption{Correlations between sessions for transitions (dark cells pertain only to calico sessions, white cells pertain only to whiteboard sessions, and lightly shaded cells are the intersection of both)}
\begin{tabular}{ p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}p{0.5cm}}
\toprule
&	C.1 &	C.2 &	C.3 &	C.4 &	C.5 &	C.6 &	W.1 &	W.2 &	W.3 &	W.4 &	W.5 \\
\midrule
C.1 	 & & & & & & & & & & & 								 \\
C.2 & 	0.82 & & & & & & & & & &								\\		
C.3 & 	0.86 &	0.88 & & & & & & & & & 							 \\		
C.4 & 	0.70 &	0.75 &	0.76 & & & & & & & &							\\	
C.5 & 	0.90 &	0.88 &	0.89 &	0.63 &	 & & & & & &						\\
C.6 & 	0.90 &	0.86 &	0.95 &	0.76 &	0.93 & & & & & &					\\	
W.1 &	0.83 &	0.83 &	0.83 &	0.45 &	0.94 &	0.84 & & & & &				\\	
W.2 &	0.45 &	0.45 &	0.66 &	0.45 &	0.46 &	0.64 &	0.44 & & & &				\\
W.3 &	0.85 &	0.84 &	0.80 &	0.46 &	0.93 &	0.83 &	0.98 &	0.41 & & &			\\
W.4 &	0.88 &	0.88 &	0.82 &	0.59 &	0.90 &	0.87 &	0.91 &	0.47 &	0.93 & &		\\
W.5 &	0.58 &	0.66 &	0.81 &	0.49 &	0.65 &	0.76 &	0.62 &	0.88 &	0.57 &	0.62 &		\\
W.6 &	0.88 &	0.88 &	0.80 &	0.54 &	0.91 &	0.86 &	0.94 &	0.42 &	0.95 &	0.97 &	0.60	\\
\bottomrule
\end{tabular}
\label{table:4}
\end{table}

Additionally, I looked for patterns of transitions. In order to discover patterns, I use the same method described in Olson et al. \citep{Olsona}. In this method, I observe for transitions that occur more often, or less often, than they are statistically expected to. In order to determine the expected value, I first calculate the number of times that a particular category occurs, and divide that number by the total number of instances of all categories, which gives us the expected percentage of occurrence. I then compare this expected percentage with the actual percentage for a particular transition. If no structure exists in the design session, then the probability of moving from one activity to the next will be the same as the expected percentage. However, if a pattern exists, then I will observe a deviation between those two numbers.

\begin{figure}
  \centering
  \resizebox{.8\hsize}{!}{ \includegraphics{./figures/CalicoVersionOne/figure14.pdf}}
\caption{Emergent pattern of transitions across both the Calico and whiteboard pairs}
\label{fig:14}       % Give a unique label
\end{figure}

Using the above method, I observed sequences of up to four transitions, and a pattern began to emerge that was present in both the Calico and whiteboard pairs. The pattern that emerged is shown in Figure \ref{fig:14}. The solid lines in Figure \ref{fig:14} represent strong trends, while the dashed lines represent transitions that are weaker, but still occur well above probabilistic levels. M stands for any management category, I for \emph{issue}, A for \emph{alternative}, and C for \emph{criteria}, in accordance with the categories in Figure \ref{fig:13}. There was a strong tendency to make frequent transitions between alternative-criteria, or between management-alternative. Additionally pairs commonly engaged in serial evaluation, where one participant would challenge the evaluation of the other, leading to new criteria with which to judge the alternative. Within the pure design categories, sequences of alternative-criteria transitions were typically punctuated by either the discussion of an issue, or a category from management. Olson et al. called these combinations IAC or MAC, based on their category names. These specific combinations are examples of ``design episodes'', which were found by them to be common in design meetings. I found these design episodes to be present in both the Calico and whiteboard pairs. The episodes of IAC, which has an expected value of .04\%, appeared to happen more commonly in Calico pairs, occurring an average of 3.25\% of all transitions, compared to 2.33\% of all transitions for whiteboard. Episodes of MAC, also with an expected value of .04\%, appeared to happen more commonly in whiteboard pairs at 2.64\% of all transitions, versus 1.61\% of all Calico transitions. 

\subsection {Satisfaction and Perceptions of Participants}
\label{results:3}

At the end of each session I verbally interviewed the participants and asked them to reflect on their own design process and satisfaction with the tool. Despite some technical difficulties, six out of eight pairs reported that they greatly enjoyed using Calico. The grid was unanimously praised as a tool to organize the design space. One participant praised that ``it was the first time [they] felt like they were using a whiteboard [in a digital medium]'', and expressed that other pen-based software tools that paginated their drawing space, such as Microsoft OneNote, made navigating designs frustrating. Five of the eight pairs explicitly praised scraps as tools for quickly copying content and moving it across canvases, however many expressed frustration that they were not able to rotate or resize them. Two pairs reported mixed feelings about the overall tool, stating that Calico merely felt like a digital whiteboard and did not compensate for the bulky hardware. All pairs reported dissatisfaction with the hardware in general, stating that it made their writing very sloppy, and that single user input was a ``deal-breaker.'' When asked if they would use Calico in their own design sessions, seven pairs expressed a great deal of interest, but only if it allowed more than one user to draw at a time. All participants reported frustration over this issue, with one participant stating that it caused them to ``spin their wheels'' while they waited for their partner to finish writing. 

Seven out of eight pairs reported that Calico did not alter or interfere with how they would normally approach a design problem, and that the ability to easily copy canvases encouraged them to explore more solutions than they normally would have. Also, pairs reported that the presence of scraps changed the way they approached creating lists, stating that they kept in mind the ability to easily copy list items and transform them into software architecture components. One pair of the eight felt that the benefits of Calico did not outweigh the hardware drawbacks, saying that they could have achieved the same design using a stack of papers. In general, nearly all participants were content with the final design that they had created, with most participants calling their final design ``a good start'' given the limited amount of time they had to create their design. 

\section{Discussion}
\label{discussion}

Building upon the results from Section \ref{results}, I now shift to interpreting what the results mean with respect to Calico use in support of software design at the whiteboard.

\subsection {Calico Features}
\label{discussion:1}

Bringing the results described in Section \ref{results} together, it is clear that Calico does provide support for subset of four design behaviors that I targeted from the total set of design behaviors of Chapter \ref{chapter:motivation} and that designers are able to use the features in support of these design behaviors when they so desire. However, it is also clear that features were not always used when they could have supported a certain design behavior and that not every pair utilized every feature.

I interpret the differences in use between the different pairs as stemming from two factors. First, some pairs had early success in using scraps, and discovered ways in which they made their life easier. They subsequently stayed with scraps more persistently. Other pairs simply drew on the canvas and ventured rarely into attempting to use scraps. With some early troublesome interactions (e.g., an accidental deletion by a slash-through, using the wrong stylus button to create a scrap), they became discouraged and used traditional methods instead. 

Second, the benefits of using scraps are not always apparent until later. One has to anticipate needing to alter a drawing by creating it using scraps in the first place, otherwise one ends up drawing it twice: once as background sketches and once to create reusable elements from parts of those sketches. This second step represents additional work, and may be skipped in favor of simply redrawing a sketch, especially if it is small or not-too-involved. 

In both cases, I believe more training and the build-up of experience over time has the potential to overcome the issue. Simply sketching on a whiteboard is so ingrained, it takes time to internalize and adjust to a new form of interaction with sketched content. 

Post-experiment conversations with the participants confirmed some of these observations. While I generally received very positive feedback, several issues were identified. First was that the nature of scraps was not as intuitive to learn as I had hoped. The participants did not think this was an outright failure, but did note the need for more training and more examples of scrap use. The participants also noted that tabs did not inform the user as to whether a neighboring canvas is occupied. Spatial navigation directly from canvas to canvas without first moving to the grid, therefore, is hampered. A simple mark to indicate a neighboring canvas has content, together with a popup on hover, may overcome this problem. 

The participants also wished for extra functionality on scraps. Some of the requested functionality is generic in nature and straightforward to add, such as rotation, resize, and different types of arrows. Some, however, is more specific and pertains to the fact that designers mentally assign meaning to scraps and expect functionality commensurate to that meaning. Such expectations arose later in the sessions, when the design had been largely worked out and the designers now wished to provide additional detail to complete the design. For instance, they asked for functionality to refine scraps into UML classes with a name, variables, and functions, or to turn scraps into lists for easy reorganization and requirement tracking. I believe such refinement can be integrated without disturbing scraps' present sketchy, informal nature, which is so crucial in the early stages of design. Particularly, I envision typing of scraps, with the assignment of type (e.g., UML class, UI element, architectural component, ER element) signaling additional functionality becoming available. 

\subsection {Design behaviors}
\label{discussion:2}

With respect to how pairs approached the design task, the Calico pairs exhibited some very similar behaviors, while the whiteboard pairs noticeably differed at times. For example, not all whiteboard pairs used transient diagrams, some maintained all of their diagrams for the entire session and refined these over a period of time. In contrast, other whiteboard pairs continuously erased what they drew and often worked from memory. Some pairs chose to work together the entire time, while other pairs worked on their design siliently in parallel at extreme ends of the board. One of the pairs that did this eventually did review each others' design and reconciled their different approaches, while another pair largely operated independently the entire session with few design decisions being contested or brought together. The Calico pairs, however, seemed to be implicitly guided into certain behaviors by the tool, and these behaviors were consistent across the majority of Calico pairs. The abundance of space led pairs to save and evolve their diagrams over time to a much greater extent. Scraps made it possible to reuse, as well as refine, diagrams, which nearly all pairs did. Additionally, the grid caused the pairs to create clear divisions between different aspects of their design. Overall, it appears that Calico's features led pairs towards the specific behaviors of design that I planned for, while the whiteboard pairs, who did not have these features, varied much more greatly in their approaches. Whether or not these behaviors lead to a positive impact on the overall design is yet to be determined, but, regardless, my observations point to the conclusion that these behaviors are supported by Calico.

\subsection {Interference}
\label{discussion:3}

Another important result of the study is that Calico did not interfere with the design activity. If anything, it led to more issues, alternatives, or criterias that were discussed. In the exit interviews, all designers agreed that Calico did not much effect the nature of their design conversations, and the results from the analysis of the design structure tend to support that. They dedicated relatively the same amount of time of each session discussing design activities at 62.6\% and 67.5\% for whiteboard and Calico conditions, respectively. Further, the use of Calico had little effect on the amount of time spent managing the activity, at 6.4\% and 7.3\% for the whiteboard and Calico pairs, respectively. The pairs intuitively understood how to manage their design using the grid, and so they did not need to exert additional effort in managing it. This behavior falls in line with previous research that supports the notion that designers make heavy use of spatial properties to orient themselves and organize their design \citep{Nickerson,Brooksa}. In the normal whiteboard sessions, it was not uncommon for participants to look back and forth between two diagrams on opposite ends of the board, and sometimes place a hand on one diagram while they looked at another. Participants used Calico in much the same way by rapidly moving back and forth between two canvases when considering two diagrams. In one exit interview, one pair requested the ability to view two canvases simultaneously in a juxtaposed view to allow a similar activity. 

Calico did, however, seem to cause some changes. The Calico tool seemed to force a greater degree of homogeneity between the pairs. The pairs that used Calico had a higher amount of conformity between them with respect to the total time spent in each category than the whiteboard pairs. The correlations between pairs for transitions between categories demonstrated the same tendency. This discrepancy in design conversations between the Calico and whiteboard pairs seems to, by and large, be related to the amount of time spent working independently. Two whiteboard pairs, 2 and 5, spent a significant portion of the session working quietly in parallel, which is reflected in their low correlation with other pairs for time spent in Table \ref{table:3}. Interestingly, the transition frequency correlation matrix in Table \ref{table:4} for these two pairs shows the same pattern of discrepancy. This suggests two things: (1) the pairs that chose to spend a significant amount of time working silently in parallel had a different design conversation structure than those whiteboard pairs that did not spend much time working silently, and, conversely, (2) the pairs that did not spend significant time working independently had a sequential process that correlated relatively highly with the Calico pairs. This second point is interesting because it shows that, while many of the Calico pairs considered themselves handicapped by the inability to work independently, the structure of their design process was similar to the highly collaborative whiteboard pairs. 

\subsection {Focus}
\label{discussion:4}

Finally, after comparing the videos with the coded transcripts, it seemed that the pairs that used Calico had a greater amount of focus on their partner's ideas and a slightly better shared understanding of recorded ideas than the whiteboard pairs. While both the Calico and whiteboard pairs spent a relatively similar amount of total time per session discussing \emph{issues}, \emph{alternatives}, and \emph{criteria}, the Calico pairs had a larger average of \emph{consecutive time per alternative and criteria} discussed. The average length of continuous time that an \emph{alternative} was 50\% longer in Calico pairs than whiteboard pairs (13 seconds versus 8.7 seconds) and the average length of continuous time a criteria was expressed was also nearly 50\% longer in Calico pairs than whiteboard pairs (11 seconds versus 7.5 seconds). While this was, in part, due to the single input interface of the electronic whiteboard, other factors in Calico forced the focus of participants well. Within Calico, the pairs would partition their design into different canvases in the grid, and so the drawing spaces in Calico tended to have a tighter focus than the whiteboard, which shows all of the content at once. As a result, the participants in the Calico pairs were not distracted by drawings within other parts of the design, whereas the whiteboard pairs could gaze at any part of their design at any time. When a participant in a Calico pair did refer back to other parts of the design, they had to switch to an entirely different canvas via the grid, effectively forcing their partner's attention to move with them to that part of the design. While Calico pairs were not able to move between topics as quickly as whiteboard pairs, it led to more elaborate explanations and longer individual responses when presenting a \emph{criteria}. Within the whiteboard pairs, it would sometimes be the case that while one partner would be explaining their solution, the other would ignore them while they considered another part of the design. The split attention was reflected in the longer time spent clarifying ideas for the whiteboard pairs, which was 53\% longer than Calico pairs (4.3\% of the total session for whiteboard pairs, on average, versus 2.8\% of the total session for Calico pairs, on average). 

\subsection {In Sum}
\label{discussion:6}

Overall, my detailed analysis of how Calico was used and influenced the design behaviors and conversations, as well as the positive responses to Calico from the participants, show that Calico has promise in enhancing design at the whiteboard. More elaborate training is likely needed for users to properly take advantage of scraps, but I still saw that Calico's features are helpful, while avoiding impeding the design process. 

Calico's strength lies in its support for the informal process of design, during which the emphasis is placed more on unstructured exploration and less on a precise analysis of the design. Some pairs benefited more than others from Calico's features, but overall, Calico assisted them in using an effective design process in which their natural behaviors were supported with explicit features. Most importantly, they were able to use the grid to partition their design effort, scraps to create diagrams with easily manipulable elements, and, occasionally, the palette to reuse impromptu design languages. Most used the grid only for working with content across multiple spaces, but some benefited from the palette as well, such as when they reused scraps on multiple canvases. Calico's strengths also include supporting the individuals in focusing on each other during the design session, as shown by my conversation analysis. 

The tradeoff to Calico's support for informal design is that the resulting designs remain sketches, and cannot be analyzed or used in ways that more formal design tools support. This kind of tradeoff is a recognized problem with informal tools, and overcoming it is becoming an active area of research \citep{Ossher2}. Another observed weakness is that Calico's interface is so different, that it takes users significant time to become familiar. Many of its benefits require the user to think ahead, otherwise they may not be using Calico optimally. Scraps in particular have a delayed pay-off, which led some pairs to not use them since they were not needed in the moment. Finally, Calico's single user input limits how a group of people may use the tool. While it improves focus, it can effectively become a bottleneck to expressing ideas if the participants want to temporarily work in parallel, a behavior which I saw put to both good and bad use in the whiteboard pairs.

\section{Theats to Validity}
\label{threatstovalidity}

Several threats to validity exist that I must keep in mind when considering my results. First, with respect to construct validity, the participants were limited to a single two-hour time span and had no access to others. Real design typically happen over longer periods of time and involve many different stakeholders. Despite this difference, during these longer periods of time, design at the whiteboard certainly happens \citep{cherubini2007let}. While my work will not support all of a design, it usefully support that slice of design that happens then and there.

Second, I must consider threats to internal validity, i.e., the factors that affect my ability to claim cause-effect relationships from the results. The single user input limitation and therefore the prevention of parallel work, is the primary concern. Exit interviews confirm this as participants in the Calico pairs felt handicapped by this limitation, and given the opportunity they would have preferred parallel work. On the other hand, results from my analysis also show that Calico pairs spent a greater amount of uninterrupted time explaining alternatives and criteria to each other, building greater cohesion. Moreover, Calico sessions still exhibits significant similarities with the non-Calico sessions in the ways the designers worked. 

Third, the conditions of the experiment may pose a risk to its external validity, i.e., its ability to generalize to what people do. In order to use Calico to its maximum potential, it would require a longer period of time than what was available in the experiment in order for people to become familiar with its features and power. Another factor was that the participants in the study were all graduate students, and so there is a chance that they may not represent the behaviors of experienced professional software designers. Novice designers have a greater tendency to become fixated on a design decision prematurely \citep{Ball}, and novices tend to mismanage their time by focusing on a single issue for an extended period whereas professional designers know when to move on \citep{Baker}. Given that the comparative nature of my experiment focused on Calico versus non-Calico design, and not on expertise, I did not further examine this factor.

Despite the fact that the experiment has its differences with real-world design, it is interesting to note that there is a relationship with how real designers work. The study conducted by Olson et al. \citep{Olsonb} demonstrated that a design session conducted in a controlled laboratory experiment is representative of a design session as it happens in the field with professional designers. Since my study uses the same measurement as those from \citep{Olsonb} and also has a high correlation with their results (\emph{r = .85}), by transitive logic, I can say that the study performed here is representative of a design session as it occurs in the field.

\section{Summary}
\label{conclusions}

In this chapter, I have presented Calico Version One, a sketch-based software design tool that provides support for the targeted subset of four design behaviors of software designers from Chapter \ref{chapter:motivation} when they work at the whiteboard. Using Calico, software designers were able to fluidly create, manipulate, and explore a design problem and its possible solutions. They did so with four features, including scraps, a grid, a palette, and gesture-based input to enhance the experience of whiteboard software design. The usefulness of these features was demonstrated in a multi-pronged, detailed evaluation, with participants in the experiment using Calico's advanced features and exhibiting the kinds of design behaviors toward which Calico was designed. Lastly, a rigorous analysis of the design conversations demonstrated that Calico not only does not interfere with the design activity, but also may lead to more shared focus between participants and longer discussions of various aspects of the designs.

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 
% \newpage \chapter{Notations}
\label{chapter:notation-paper}

\section{Background}

Lorem ipsum 

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 
 \newpage \chapter{Calico Version Two}
\label{chapter:calico-version-two}

Equipped with the lessons from Chapters \ref{chapter:calico-version-one} and \ref{chapter:notation-paper}, I have gone through an iterative process of designing, implementing, and evaluating various possible incarnations of Calico. Through experiments and trials with the technology over several years, including using the tool in my own meetings, deploying it in the classroom \cite{Loksa2013}, and formal experimentation \cite{mangano2012design}, I have slowly, but surely, developed an understanding of how to mesh more advanced functionality with the fluidity that the whiteboard experience demands. I have specifically refined what first was a broad range of exploratory features into a small, cohesive set of just five features that form the conceptual core of Calico.

The laboratory evaluations in Chapter \ref{chapter:calico-version-one} have given insight into the usefulness and shortcomings of the features of Calico Version One, such as scraps, the palette, and the grid. Participants found scraps useful to manipulate content, but found them difficult to use as representations for class diagrams because scraps could not be resized or rotated. Similarly, participants found the palette useful as a global clipboard to transfer content across canvases, but the inflexible nature of scraps made reusing them from the palette difficult. The grid itself proved to be a highly used feature in navigating and organizing sketches, but participants reported wanting more organizational power such as grouping sets of canvases into topics.

The design sessions with professional software designers in Chapter \ref{chapter:notation-paper} revealed greater insight into the design activities that were not captured in Chapter \ref{chapter:calico-version-one}. Regarding the representations professional designers used, lists were ubiquitous in supporting several aspects of design, including the user interface, software architecture, and requirements. The professional designers also demonstrated several methods of refining and evolving their sketches, including refining them in place by boxing items in lists, or redrawing sketches in place. Further, the designers often did not edit their sketches during activities such as mental simulations or reviewing requirements, but instead simply pointed at sketches the majority of the time while they talked.

From the insights found in Chapter \ref{chapter:calico-version-one} and Chapter \ref{chapter:notation-paper}, I have arrived at the final five features in this chapter, which I believe to be effective in supporting the fourteen design behaviors identified in Chapter \ref{chapter:motivation}. For reference, Table \ref{table:calico-version-two:designbehaviors} summarizes the design behaviors here in this chapter, for reference. The features in this chapter represent the final iteration that will be evaluated in Chapter \ref{chapter:evaluation}.

In this chapter, I first describe the motivation that lead to the changes I made to each of the Calico Version One features and to the introduction of new ones. The motivation is explained in terms of the design behaviors and the final feature set is mapped to their motivating design behaviors. Next, I describe the features in detail, eliciting the exact functionality for each feature. Lastly, I describe the implementation details of the system.

\section{Updated features}
\label{chapter:calico-version-two:features}

In this section, I review the changes between Calico Version One and Calico Version Two, as well as the motivations that lead to these changes. I do not go into great detail for the features here, but only refer to the features at a high level. Section \ref{chapter:calico-version-two:implementation} describes the features in detail.

\subsection{Changes to basic sketching}

With respect to basic sketching, Calico Version Two includes added functionality that enriches the basic sketching experience of Calico Version One. The first fundamental change to Calico is its transition from a single client application to a distributed, synchronous sketching application. One of the greatest challenges encountered by participants in Chapter \ref{chapter:calico-version-one} was the inability to perform asynchronous work. Given the constraints of the technology involved with the Hitachi Starboard FX-DUO, we chose to reimplement Calico as a distributed application so that users on separate machines could sketch at the same time. The inclusion of distributed, synchronous sketching better supports design behavior 12, in which designers switch between synchronous and asynchronous work. Several other small features were included as well, such as the ability to import images and email canvases, which help to bring outside work into Calico.

\subsection{Changes to scraps}

With respect to scraps, much of the interaction changed between Calico Version One and Calico Version Two based on our feedback in Chapter \ref{chapter:calico-version-one}. First is the introduction of selection scraps (center of Figure \ref{fig:calico-version-two:overviewa}). In Chapter \ref{chapter:calico-version-one}, participants in the user evaluations often used scraps for single actions, such as moving, and then would ``drop'' the scrap after moving its content. Based on this usage, the behavior of scraps was changed such that newly created scraps instead function as a lasso, and will disappear unless they are ``pinned'' by the user, as in Figure \ref{fig:calico-version-two:overviewa}. This behavior, termed selection scrap, increases the fluidity of using scraps by reducing the amount of actions needed to perform common actions such as moving sketches. 

Second is the functionality to transform existing sketches into scraps. Participants in the user trials of Chapter \ref{chapter:calico-version-one} reported that they sometimes did not use scraps because it required them to plan ahead. That is, they did not know early on some areas would need to be an object later. Retroactively ``promoting'' such an area to a scrap lead to objects with double boundaries. To solve this, this second behavior addresses this issue and further supports refinement of sketches (design behavior 3) by allowing the user to create scraps when they need them without as much need to plan ahead. 

Third, the menu for scraps was revised to appear as a ``bubble menu'' (Figure \ref{fig:calico-version-two:overviewa}), which presents the buttons on the periphery of the scrap. Users of Calico Version One head difficulty ``hitting'' the menu button, which was small, and also stated that the button and menu contents occluded the visibility of the scraps themselves. Items in the bubble menu are much easier to tap on and do not occlude associated scrap contents. Furthermore, it was much easier to add new previously requested features such as resize and rotate, since the bubble menu has room to accommodate additional items. The bubble menu further replaces much of the functionality enabled by the gesture system in Calico Version One. Finally, in exit interviews for the laboratory evaluation of Chapter \ref{chapter:calico-version-one}, participants reported that they did not use a number of features, because they were not discoverable through the set of gestures. The bubble menu instead uses graphical icons to offer this functionality. Gestures have largely been removed from Calico Version Two, with the exception of the press-and-hold gesture.

Fourth, connectors were revised such that they could also be created from existing sketches, matching the revised behavior of scraps, and also now retain their shape (as shown in Figure \ref{fig:calico-version-two:overviewa}). Preserving their shape provides the user with greater control in how they express their arrow, improving their ability to draw different types of connectors (design behavior 1). 

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/CalicoVersionTwo/overview-scraps}
  \caption{Regular sketches can be transformed into selection scraps, which can further be made permanent by ``pinning'' the scrap.}
  \label{fig:calico-version-two:overviewa}
\end{figure*}

Additionally, specialized scraps, such as text-scraps and list-scraps, were introduced. Chapter \ref{chapter:notation-paper} demonstrated the importance of creating and managing lists of text alongside diagrams. Professional designers created, referred to, and checked off items in lists throughout their design sessions. Participants in the laboratory evaluation from Chapter \ref{chapter:calico-version-one} also devoted entire canvases to writing out requirements as plain text. Text-scraps and list-scraps target these actions specifically by automatically organizing scraps for the user.

Despite these major chances to the interactions of scraps, much of their fundamental nature remains the same in order to continue supporting the same design behaviors. Scraps still retain the shape of the pen stroke used to create them. Parenting is enabled by stacking scraps on top of one another to implicitly group them. Both of these functionalities contribute towards supporting the first and second design behaviors.

\subsection{Changes to palette}

With respect to the palette, its basic operation has changed little. The palette continues to serve its purpose as a global source to save scraps, which can immediately be reused anywhere by dragging it back to the canvas from the palette. However, the palette does include updates that allow the user to create multiple palettes of scraps that they can move between or delete. 

The palette has, however, been greatly enriched by the other new features. The revised distributed nature of Calico allows the items in the palette to be automatically shared by everyone who is connected to the same server. The ability to resize and rotate scraps, as well as the functionality to save scraps with imported images into the palette, increase the ability to create and reuse scraps pertaining to a project. They, in particular, increase the palette's ability to support the creation of impromtu notations (design behavior 4).

\subsection{Removal of grid and introduction of intentional interfaces}

With respect to the grid, the feature has been replaced by intentional interfaces. While the grid has proven useful in past evaluations, intentional interfaces not only enable the user to organize their canvases like the grid, but also provide the user with greater ability than with the grid. Here I review intentional interfaces, as well as the shortcomings of the grid that intentional interfaces address. 

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/CalicoVersionTwoCluster}
  \caption{Intentional Interfaces - Intention View.}
  \label{fig:calico-version-two:overviewb}
\end{figure*}

Intentional interfaces is composed of three pieces. First, rather than a grid layout, intentional interfaces organizes canvases into a radial layout (Figure \ref{fig:calico-version-two:overviewb}), which can further be organized by branching. This functionality was created to help provide a greater grouping mechanism for canvases. While the grid did serve as an effective and straightforward mechanism to organize canvases within a design session, in practice, the grid has several shortcomings that merited a more complex feature such as intentional interfaces. For example, the participants in the laboratory evaluation in Chapter \ref{chapter:calico-version-one} grouped canvases of similar topics, such as requirements of interface mockups, in close proximity to one another. In exit interviews, participants expressed that they desired to go one step further with explicit grouping. 

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/CalicoVersionTwo/overviewb-tags}
  \caption{Canvases are related to one another within intentional interfaces using tags.}
  \label{fig:calico-version-two:overviewb-tags}
\end{figure*}

In the second piece of intentional interfaces, each canvas has a tag that explicitly associates it with the previous canvas it branches out from, i.e., it is an ``alternative'', ``level of abstraction'', and so on, of the previous canvas. Figure \ref{fig:calico-version-two:overviewb-tags} depicts two canvases associated in this manner, in which one canvas is marked as an ``alternative'' of another. In Chapter \ref{chapter:calico-version-one}, the participants were observed to shift between perspectives of the same entity. In Chapter \ref{chapter:notation-paper}, the professional designers demonstrated several explicit reasons for navigating between canvases, such as shifting between perspectives, abstractions, or alternatives. Tags enable the user to declare their intention when creating a canvas, thus providing a lightweight mechanism for supporting many of the navigation design behaviors 5, 6, and 7 in Section \ref{chapter:motivation:navigation}. 

The third piece is the breadcrumb bar (top-left of Figure \ref{fig:calico-version-two:overviewb}), which allows designers to navigate through the canvases. This third feature enriches the radial layout with an additional method to navigate between the canvases, and displays the associated tags as well.

\subsection{Introduction of fading highlighter}

In addition to the previous features, a new feature, the fading highlighter, is introduced to support the explanation of sketches (design behavior 13). The highlighter separates the act of highlighting from drawing. This functionality allows the designer to point, circle, and draw sequences over diagrams, but the strokes made by the designer fade a few seconds after they are made. While a small piece of functionality, the highlighter serves an important role. It allows a group of designers, and even the individual designer, to perform mental simulations (design behavior 8) and reviews of progress (design behavior 9), both while at the same whiteboard and over the network in distributed cases. The professional designers in Chapter \ref{chapter:notation-paper} demonstrated that in nearly 80 percent of reasoning activities, they do not manipulate the sketch itself, but rather only look at or gesture over a sketch. The highlighter enables users to accomplish the same purpose of gesturing over diagrams without modifying their sketches, yet be much more explicit in expressing their intent.

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=8cm,keepaspectratio]{./figures/CalicoVersionTwo/highlighter}
  \caption{Fading highlighter.}
  \label{fig:calico-version-two:overviewd}
\end{figure*}

%\begin{figure}%
%  \centering
%  \subfigure[Scraps] {
%      \label{fig:calico-version-two:overviewa}
%	  \includegraphics[width=7cm,keepaspectratio]{./figures/CalicoVersionTwo/overview-scraps}
%   }
%  \subfigure[Intentional Interfaces - Intention View] {
%      \label{fig:calico-version-two:overviewb}
%      \includegraphics[width=7cm,keepaspectratio]{./figures/CalicoVersionTwoCluster}
%   }
%  \subfigure[Palette] {
%      \label{fig:calico-version-two:overviewc}
%      \includegraphics[width=7cm,keepaspectratio]{./figures/CalicoVersionTwo/overview-palette}
%   }
%  \subfigure[Highlighter] {
%      \label{fig:calico-version-two:overviewd}
%      \includegraphics[width=7cm,keepaspectratio]{./figures/CalicoVersionTwo/highlighter}
%   }
%   \caption {Calico features}
%   \label{fig:calico-version-two:overview}
%\end{figure}%

\subsection{Features and design behaviors they support}

With the changes and additional features introduced in this section, Calico now supports all fourteen design behaviors. Table \ref{table:calico-version-two:designbehaviors} provides a mapping between the design behaviors and the features that support them, as well as the reason why I believe the features support the respective design behavior.

\begin{center}
\begin{longtable}{|p{4cm}|p{4cm}|p{4cm}|p{4cm}|}
\caption{The set of design behaviors and the features that support them}\\
\hline
\textbf{Design Behavior} & \textbf{Supporting Feature} & \textbf{Design Principles} & \textbf{Reason} \\
\hline
\endfirsthead
\multicolumn{4}{c}%
{\tablename\ \thetable\ -- \textit{Continued from previous page}} \\
\hline
\textbf{Behavior} & \textbf{Supporting Feature} & \textbf{Design Principles} & \textbf{Reason} \\
\hline
\endhead
\hline \multicolumn{4}{r}{\textit{Continued on next page}} \\
\endfoot
\hline
\endlastfoot
\multicolumn{4}{|c|}{\textbf{Kinds of sketches software designers produce}} \\
\hline
1. They draw different kinds of diagrams	&Scraps \& connectors	&User-drawn shapes are preserved; strokes drawn inside scraps are grouped implicitly; scraps \& connectors can be hierarchically composed	&Scraps \& connectors provide a unified abstraction for informally modeling a range of notations\\
\hline
2. They produce sketches that draw what they need, and no more	&	&	&\\
\cline{1-4}
2a. They only draw what they need w.r.t. the design at hand	&Scraps \& connectors	&Calico, at its core, acts just like a whiteboard, not dictating any content	& %\multirow{2}{4cm}{
Can draw just what they want and nothing more
%} 
\\
\cline{1-4}
2b. They use only those notational conventions that suit drawing what they need	&Scraps \& connectors	&Scraps \& connectors do not impose any notational conventions or uses &	Can draw just what they want and nothing more\\
\hline
3.      They refine and evolve their sketches over time	&	&	&\\
\hline
3a.   They detail their sketches with increasing notational convention	&Scraps \& connectors&Provide additional visual structure and behaviors to scraps and connectors 	&General scraps \& connectors can incrementally be refined to look like and behave as different notations\\
\cline{1-4}
3b.   They appropriate a sketch in one notational convention into another notational convention	&Scraps \& connectors&Visual structure \& behaviors of scraps and connectors can be changed 	&Scraps and connectors looking like and adhering to one notation can be changed into the look and behavior of another\\
\cline{1-4}
4.      They use impromptu notations	&Scraps \& connectors	&Any visual convention can be adopted simply by drawing similarly shaped scraps	& New notational conventions can be introduced, used as any other, and reused\\
\cline{2-3}
	&Palette	&Any scrap, connector, or set of them can be stored in a palette for later reuse	&\\
\hline
%	&Scraps and connectors &Elementary visual looks and behavior can be randomly composed	&\\
%\hline
\multicolumn{4}{|c|}{\textbf{How they use the sketches to navigate through a design problem}} \\
\hline
5.      They move from one perspective to another	&Scraps \& connectors	&They can mix and match different notational conventions on a single canvas	&Different perspectives can be developed within a canvas	\\
\cline{2-4}
	&Intentional Interfaces	&Users can explicitly request a new canvas to work on a perspective	& Canvases with different perspectives are explicitly related\\
\hline
6.      They move from one alternative to another	&Scraps \& connectors	&Different alternatives can be quickly constructed by copying and moving and otherwise manipulating scraps and connectors	&Different alternatives can be developed within a canvas\\
\cline{2-3}
	&Palette	&Different alternatives can be quickly constructed by reusing elements from the palette and composing them differently &\\
\cline{2-4}
	&Intentional Interfaces	&Users can explicitly request a new canvas to work on a different alternative	&Canvases with different alternatives are explicitly related\\
\hline
7.      They move from one level of abstraction to another	&Scraps \& connectors	&Different abstractions can be quickly constructed by copying and moving and otherwise manipulating scraps and connectors	&Different levels of abstraction can be developed within a canvas\\
\cline{2-4}
	&Intentional Interfaces	&Users can explicitly request a new canvas to work on a deeper level of abstraction	&Canvases with different abstractions are explicitly related\\
\cline{1-4}
8.      They perform mental simulations	&Highlighter	&Users can use the highlighter to mark up their diagrams without editing them	&The design can be gestured at without modifying it\\
\hline
9.      They juxtapose sketches	&Scraps	&Uses can move perspectives, alternatives, and abstractions next to one another by moving scraps	&The pieces of the design can better be compared without looking across the canvas\\
\cline{2-4}
	&Intentional Interfaces	&Users can compare canvases by zooming in on a portion of the intentional view	&Content across canvases can be more easily juxtaposed without manipulating content within canvases\\
\hline
10.  They review their progress	&Intentional Interfaces	&Users can step back and examine their progress and process, overall and in parts, in the intention view	&The tags and visual structure provide some insight into the rationale of canvases\\
\hline
11.  They retreat to previous ideas	&Intentional Interfaces	&Users can choose to enter one canvas in the intentional view or make a new canvas at any time	&Users can return to older versions they kept\\
\hline
\multicolumn{4}{|c|}{\textbf{How they collaborate on them}}\\
\hline
12.  They switch between synchronous and asynchronous work	&Intentional Interfaces	&Users can choose to enter one canvas in the intention view, or make a new canvas and work separately	&Work is synchronous while in one canvas, asynchronous while in different canvases\\
\hline
13.  They explain their sketches to each other	&Highlighter	&Users can use the highlighter to draw attention to certain parts of a canvas	&Draw attention of designers to highlighted area\\
\hline
14.  They bring their work together	&Palette	&Designers can place sketches from different canvases into palette and later merge them into a single canvas		&Users can return to older versions they kept\\
\cline{2-3}
	&Intentional interfaces	&Users can navigate back and forth between canvases, and generate new ones&
\label{table:calico-version-two:designbehaviors}
\end{longtable}
\end{center}



\section{Implementation}
\label{chapter:calico-version-two:implementation}

In the previous section, I motivated the changes to existing features and the introduction of new features for Calico Version Two. In this section, I explain the \emph{how} for each feature, explaining how each feature works in detail.

\subsection{Implementation of basic sketching}

Figure \ref{figure:calico-version-two:canvas} presents Calico as it first appears when a designer enters a canvas. Users can immediately draw or write by dragging their pen across the whiteboard. Just as on a standard whiteboard, they can make any marks they wish, anywhere, in any shape. The drawing canvas is largely clear of any obstructions, with features available on the periphery to maintain the appearance of a standard whiteboard. The side panels are mirrored on both the left and right sides, and a panel at the bottom displays status messages and has a minimal set of auxiliary buttons. 

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/CalicoVersionTwo/CalicoVersionTwoCanvas}
  \caption{The Calico canvas interface, including the side panels and the drawing space.}
  \label{figure:calico-version-two:canvas}
\end{figure*}

The side panels have an assortment of features to help the user while sketching. The green icons belong to the features that affect the entire set of sketches. The first five icons pertain to intentional interfaces described below, and help the user in creating and navigating canvases. The next icon allows the user to erase the canvas. The user can undo and redo their actions. The other icons pertain to the contents drawn on the canvas. The buttons enable functionality such as pen color, stroke widths, pen modes, and special types of scraps. The pen mode toggles the input mode between regular sketching, the eraser that removes regular strokes, and the highlighter.  

\subsection{Implementation of scraps}

As mentioned in Section \ref{chapter:calico-version-two:features}, scraps enable users to both manipulate content and create informal diagrams as in Figure \ref{fig:calico-version-two:scrapse}. In order to support users in these activities, scraps have developed a unique set of functionality. Scraps can be created while sketching using the scrapping gestures in Figure \ref{fig:calico-version-two:scrapsa} and \ref{fig:calico-version-two:scrapsb}, which transform drawn content into the selection scraps in Figure \ref{fig:calico-version-two:scrapsc} for quick manipulation of sketches, or into the regular scraps in Figure \ref{fig:calico-version-two:scrapsd} for representing diagrams. Once made into a regular scrap, it becomes an implicit group that is universally manipulatable, stackable, and relatable, as in Figure \ref{fig:calico-version-two:scrapse}. We review these functionalities one by one.

\subsubsection{Scrapping gestures} Calico has two basic gestures that it uses for creating scraps directly while drawing. The first gesture creates a scrap by circumscribing an area and releasing the pen in a landing zone, as seen in Figure \ref{fig:calico-version-two:scrapsa}. In order to not interfere with the sketching activity itself, the landing zone will only appear if the stroke is sufficiently long enough. The second gesture for creating a scrap is done by pressing-and-holding the pen inside a stroke that is already circumscribing an area, which animates a dotted red circle and creates a scrap from the enclosing stroke, as in Figure \ref{fig:calico-version-two:scrapsb}. The second gesture allows the user to create a scrap after the stroke has already been created, either as a recovery mechanism if the user missed the landing zone of the first strategy, or if they choose to convert an already existing sketch into a scrap. The ability to create scraps from plain sketches helps support the refinement of sketches (design behavior 3).

\begin{figure}%
  \centering
  \subfigure[Landing zone scrapping gesture] {
      \label{fig:calico-version-two:scrapsa}
	  \includegraphics[width=7cm,keepaspectratio]{./figures/CalicoVersionTwo/scrapgestures-a}
   }
  \subfigure[Press-and-hold scrapping gesture] {
      \label{fig:calico-version-two:scrapsb}
      \includegraphics[width=7cm,keepaspectratio]{./figures/CalicoVersionTwo/scrapgestures-b}
   }
  \subfigure[Selection scrap] {
      \label{fig:calico-version-two:scrapsc}
      \includegraphics[width=5cm,keepaspectratio]{./figures/CalicoVersionTwo/scrapgestures-c}
   }
  \subfigure[Scrap] {
      \label{fig:calico-version-two:scrapsd}
      \includegraphics[width=5cm,keepaspectratio]{./figures/CalicoVersionTwo/scrapgestures-d}
   }
   \subfigure[Stacked scraps with connectors] {
      \label{fig:calico-version-two:scrapse}
      \includegraphics[width=14cm,keepaspectratio]{./figures/CalicoVersionTwo/scrapgestures-e}
   }
   \caption {Scrap functionality}
   \label{fig:calico-version-two:scraps}
\end{figure}%

\subsubsection{Selection and regular scraps} Scraps created from the pen gestures are first made into selection scraps (Figure \ref{fig:calico-version-two:scrapsc}), which disappear after the scrap loses focus. Upon creation, the selection scrap highlights any content inside of it, and allows the user to manipulate the selected contents using the bubble menu surrounding the scrap, as seen in Figure \ref{fig:calico-version-two:scrapsc}. When the selection scrap loses focus, it immediately disappears and returns its contents to the canvas. If the user wishes to permanently retain the scrap, they can tap any of the two scrap icons in the upper left of the bubble menu, which transforms it into a regular scrap, as in Figure \ref{fig:calico-version-two:scrapsd}. Selection scraps allow users to benefit from the manipulation capabilities of scraps without forcing the contents of their sketch to be scraps.

\subsubsection{Implicit grouping} Scraps build upon the approach taken in Translucent Patches \cite{Kramer}, which allows users to explicitly declare an area as a group. Anything that is either entirely circumscribed in the first place or otherwise written or drawn in this area afterwards is automatically part of the group. Consider the sketch of ATM in Figure \ref{fig:calico-version-two:scrapse}. It was first drawn on the canvas, then circumscribed by the stylus to become a scrap. The scrap is now a persistent object with a blue background. Any further additions to the ATM scrap, or any other scrap in Figure \ref{fig:calico-version-two:scrapse}, automatically become part of that scrap.

\subsubsection{Manipulation} Scraps are movable, copy-able, rotatable, and resizable. Tapping or writing on a scrap immediately highlights it and presents the bubble menu on its periphery, as in Figure \ref{fig:calico-version-two:scrapsc} and \ref{fig:calico-version-two:scrapsd}. These seemingly innocuous actions in reality represent a significant improvement over the standard whiteboard: content drawn can be rapidly reorganized. It particularly is important that such reorganization takes place in the language of the user: elements that they have deemed of sufficient importance to promote to being a scrap are the elements that are manipulated.

\subsubsection{Stacking} Moving a scrap to a position where it is entirely overlapped by another scrap attaches it to the scrap behind it, allowing users to quickly create a stack of scraps (thereby creating hierarchically composed groups), as one would a pile of papers. For instance, the scraps labeled ``Deposit'', ``Withdrawal'', and ``CheckBalance'' in Figure \ref{fig:calico-version-two:scrapse} are part of the scrap labeled ``Transactions''. If ``Transactions'' is moved, ``Deposit'', ``Withdrawal'', and ``CheckBalance'' are moved as well. Dragging a scrap off of another scrap un-groups it. Moving the scrap labeled ``Deposit'' from its current location will ungroup it from ``Transactions'' and, in one fluid motion, group it with ``User Interface'' when the user drops it there. Note that dragging a scrap implicitly moves it to the top of the order of scraps; scraps do not slide under other scraps.

\subsubsection{Connectors} By dragging the digital stylus from one scrap to another, the pen stroke becomes highlighted and presents the user with an icon to transform that stroke into a connector. Alternative, as with scraps, connectors can be created retroactively by press-and-holding a stroke that begins and ends in a scrap, and then tapping the ``create connector'' button. The connector preserves the shape of the stroke, but is decorated with an arrowhead. The connector is persistent and anchored to the places where it originated and ended. When scraps are moved, the connectors move accordingly and keep the two scraps related. In Figure \ref{fig:calico-version-two:scrapse}, the scrap ``ATM'' relates to the ``Transactions'' and ``User Interface'' scraps, and the scrap ``Transaction'' relates to ``BankDB''.

\subsubsection{List-scraps and text-scraps} List-scraps, depicted in Figure \ref{figure:calico-version-two:list-scraps}, are a specialized feature that organize scraps into a linear vertical list, and changes its boundaries to fully contain the children within it. If a regular scrap has children, the user may promote the regular scrap to a list-scrap. Promoting a scrap to a list-scrap reorganizes the items contained in it to be a list. Just as with implicit grouping of regular scraps, the user can drag scraps on top of the list to automatically group them. The list reorganizes items after inserting the recently added scrap into the vertical list. Each item in a list has an associated box, which can be checked and unchecked. Further, as with regular scraps, lists can be nested within other lists to create multi-level structures.

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=8cm,keepaspectratio]{./figures/CalicoVersionTwo/list-scraps}
  \caption{List-scraps automatically organize scraps into a vertical list.}
  \label{figure:calico-version-two:list-scraps}
\end{figure*}

Text-scraps were designed to work in conjuction with list-scraps to help the user create content more quickly from the keyboard. Using either the text-scrap button on the menubar (Figure \ref{figure:calico-version-two:canvas}), or by pressing the enter key while in the canvas, the user can create a new scrap using their keyboard. If a list-scrap is already selected, the new text-scrap will be automatically appended to the end of the list.

\subsection{Implementation of palette}

The palette saves a template of a scrap that can be reused (See Figure \ref{fig:calico-version-two:palettea}). Each scrap contains a palette button on the top right of its bubble menu (Figure \ref{fig:calico-version-two:paletteb}), which adds that scrap to the palette bar. Once added, the user can reuse the scrap by pressing the pen on the image of the icon that they wish to use, and dragging it back onto the canvas. The items on the palette bar are globally available both across all canvases in the grid, and all users connected to the same server. Further, the palette has multiple sets that the user can switch between using the up and down arrows on the palette bar in Figure \ref{fig:calico-version-two:palettea}, or can also create a new set by pressing the plus icon. This enables users to manage multiple related sets of symbols. The palette bar contains several other features for managing the palette, such as creating a new empty palette, deleting a palette, and importing images into a palette. When the user is finished using the palette, they can toggle it to be invisible from the bottom menu bar, as seen in the bottom right of Figure \ref{figure:calico-version-two:canvas}.

\begin{figure}
  \centering
  \subfigure[Palette bar with scraps] {
      \label{fig:calico-version-two:palettea}
      \includegraphics[width=8.5cm,keepaspectratio]{./figures/CalicoVersionTwo/palette-a}
   }
  \subfigure[Palette icon on a scrap’s bubble menu] {
      \label{fig:calico-version-two:paletteb}
      \includegraphics[width=4.5cm,keepaspectratio]{./figures/CalicoVersionTwo/palette-b}
   }
   \caption {Scraps can be added to palette for rapid reuse}
   \label{fig:calico-version-two:palette}
\end{figure}

\subsection{Implementation of intentional interfaces}

The intentional interfaces feature presents a novel method to organizing sketches across canvases. I break the intentional interfaces feature down into the following components: the wall that organizes topics, clusters that organize canvases into a radial pattern, tags that related canvases and organize them into branches, the canvas interaction with tags, and the breadcrumb bar that aids in the navigation between canvases.

\subsubsection{The wall perspective} 

The intentional interfaces feature is organized into three levels: the wall, clusters, and canvases. The wall provides a high level mechanism for partitioning canvases.  While the grid is no longer a feature within Calico, the grid can be thought of as being equivalent to clusters, and the wall as one level of organization higher. The inclusion of the wall is motivated by long-term, multi-month usage of Calico, in which multiple people were using the same Calico grid to manage their projects. For example, Figure \ref{figure:calico-version-two:ii-wall} depicts the wall belonging to the SDCL group, which contains content that is several years old. The wall and clusters provide a convenient way to separate projects between people. In other long term usages of Calico, users that shared a Calico grid requested a separate space to work with different topics. In the wall, grouped content is segregated from other grouped content. In the grid, one person's ``space'' of canvases can cross with another person’s space of canvases. Intention interfaces eliminates that problem, helping both with navigation of canvases and people working together in the same design space. Further, canvases can now be labeled, providing additional support to organizing clusters. 

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/CalicoVersionTwo/ii-wall}
  \caption{The Calico wall belonging to the SDCL group, which contains sketches that are several years old and are used by several group members regularly.}
  \label{figure:calico-version-two:ii-wall}
\end{figure*}

From the wall, users may navigate and manage their canvases. The user may enter a cluster by tapping on the white space within a cluster, or directly into a canvas by tapping on a canvas. They may also move canvases between clusters by dragging a canvas onto a different cluster. Clusters are created and deleted automatically, where a new cluster is created when there are no more empty clusters, and extra clusters are deleted such that there remains only one empty cluster at a time.

\subsubsection{The cluster perspective}

Within a cluster, canvases are organized into a radial layout. The radial layout provides a method to organize a scalable number of canvases, where the size of the cluster is increased to accommodate more canvases. Figure \ref{figure:calico-version-two:ii-cluster} depicts the cluster perspective of ``Nick M.'s stuff'', which is the first column, second row cluster in Figure \ref{figure:calico-version-two:ii-wall}. New canvases are added to a cluster by tapping the ``new canvas'' button in the top right of Figure \ref{figure:calico-version-two:ii-cluster}. In order to help the user in mentally organizing the set of canvases in a cluster, the organization of clusters takes its inspiration from a mechanical clock. The canvases are sorted such that their order ascends in a clockwise pattern, where the first canvas is always located at the midnight position of a mechanical clock-hand, and the most recently added canvas will always appear at the end of the clock-wise radial loop, just before the midnight clock-hand position. 

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/CalicoVersionTwo/ii-cluster}
  \caption{The cluster perspective of ``Nick M.'s stuff''.}
  \label{figure:calico-version-two:ii-cluster}
\end{figure*}

Within a cluster, the user may move canvases freely. In preliminary feedback of Calico Version Two, users reported that the cluster provided them a meta-design space, and organizing the positioning of the canvases was part of their design process. In order to accomodate this action, canvases that are moved from the radial layout become ``pinned'', and will maintain their spatial position in the cluster regardless of other canvases being added or deleted from the radial layout. Canvases may further be ``unpinned'', in which case they return to the radial layout and their positioning becomes automatically managed again.

\subsubsection{Tagging} 

Canvases in the radial layout may branch outwards from the center with the use of tagging, as depicted in Figure \ref{figure:calico-version-two:ii-cluster-branch}. In the laboratory evaluations of Chapter \ref{chapter:calico-version-one}, the users often moved to a new canvas for a particular purpose, such as exploring a new perspective, diving into more detail, or exploring an alternative. Intentional interface tags formally capture these links between canvases. Upon visiting a new canvas, the user is asked to tag the new canvas with an intention (Figure \ref{figure:calico-version-two:ii-canvas}). After choosing a tag, the new canvas is linked to the previous canvas in the radial layout. The tagged canvases form a linked list in the radial layout, thereby preserving their relationships with one another. For example, Figure \ref{figure:calico-version-two:ii-cluster-branch} depicts a diagram that is refined over several canvases using tags. The canvas in the center of the circle, ``9. MiSE - Figure 2 original'', contains the original diagram that was imported as an image. The work from the center canvas is extended into ``9.1 MiSE - Figure 2 revised'', which is tagged as a ``Continuation'' of the previous canvas. The diagram is refined and revised until the final diagram was reach in canvas ``9.4 MiSE Figure 2c''.

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/CalicoVersionTwo/ii-cluster-branch}
  \caption{A diagram is refined and revised across several canvases, which are linked with ``Continuation'' and ``Alternative'' tags.}
  \label{figure:calico-version-two:ii-cluster-branch}
\end{figure*}


Tagging is introduced in order to address the ``neighbor knowledge awareness problem'' \cite{dekel2007notation}, which was observed from participants usage of the grid in Chapter \ref{chapter:calico-version-one}. In the neighbor knowledge problem, canvases that are initially located adjacent to one another typically have related information, but that relation can be lost if the canvases are moved away from one another. With respect to the grid, users typically cluster related content together, but, due to fixed spatial constraints, the grid does not scale and frequently must be manually reorganized. The radial layout in the cluster perspective, in contrast, does scale, and preserves the relationships between canvases. 

The tag panel as presented in the canvas perspective is depicted in Figure \ref{figure:calico-version-two:ii-canvas}. The tag panel is initially populated with a set of tags inspired by design behaviors 5, 6, and 7, such as ``alternative'', ``perspective'', and ``abstraction''. The user, however, may add, edit, or delete the set of tags. Tags can be selected by tapping on the tag itself, after which the tag panel is dismissed. The user may toggle the tag panel at a later point by pressing the tag button in the status bar (bottom right of Figure \ref{figure:calico-version-two:ii-canvas}).

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/CalicoVersionTwo/ii-canvas}
  \caption{The tag panel appears in the upper-right upon entering a newly created canvas.}
  \label{figure:calico-version-two:ii-canvas}
\end{figure*}

\subsubsection{Navigating canvases and breadcrumb bar} 

From the canvas perspective, the user can create, manage, and navigate their canvases within the context of intentional interfaces. In past evaluations of Calico, users often wanted a new empty canvas for a certain purpose, but did not necessarily want to go to the grid to find an empty one. Intentional interfaces reduces the cognitive burden of the previous version by removing the necessity of choosing the new canvas's location. Instead, the user creates a new empty canvas by clicking the ``new canvas'' or ``copy canvas'' buttons (depicted in Figure \ref{figure:calico-version-two:canvas}), which each take the user to a new canvas, and in the latter case, populate the new canvas with a copy of the content from the original canvas at which the user was working.

The user's recent navigation history is also recorded to help them navigate canvases. The user may use the backwards and forwards buttons (depicted in Figure \ref{figure:calico-version-two:canvas}) to navigate among the history of visited canvases without returning to the cluster or wall perspectives. When the user returns to the cluster or canvas perspective, the most recently visited canvas is highlighted with a blue halo (as in Figure \ref{figure:calico-version-two:ii-cluster-branch}).

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/CalicoVersionTwo/ii-canvas-breadcrumbs}
  \caption{The breadcrumb bar in the upper-left enables users to navigate between canvases without using the wall or cluster perspectives.}
  \label{figure:calico-version-two:ii-canvas-breadcrumbs}
\end{figure*}

The breadcrumb bar at the top of the canvas, depicted in Figure \ref{figure:calico-version-two:ii-canvas-breadcrumbs}, further helps the user in navigating between their canvases. Visible in both the canvas and cluster perspectives (but not the wall perspective), the user can tap on a canvas to reveal its associated canvases. From these canvases, the user can further tap again to directly navigate to any canvas, cluster, or the wall. For example, in Figure \ref{figure:calico-version-two:ii-canvas-breadcrumbs}, the user has tapped ``9. Mise - Figure 2 original'', which revealed a drop-down that displays canvases 9.1 through 9.4 (all of which are also depicted in Figure \ref{figure:calico-version-two:ii-cluster-branch}.

\subsection{Implementation of highlighter}

The fading highlighter enables the user to draw on the canvas without modifying its contents. The fading highlighter creates a dark-golden stroke that is 3px wide, and begins to disappear after 4 seconds until the stroke created with the fading highlighter is deleted. The strokes created with the fading highlighter are visible to everyone connected to the same Calico server and using the same canvas, and the stroke will also disappear and be deleted in the same intervals on remote machines. The fading highlighter supports users in performing mental simulations (design behavior 8), reviews of progress (design behavior 9), and explaining sketches to other designers (design behavior 12), particularly if the other designer is participating in the discussion over a network in a remote location.

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=9cm,keepaspectratio]{./figures/CalicoVersionTwo/highlighter}
  \caption{Highlighter}
  \label{figure:calico-version-two:highlighter}
\end{figure*}

\section{Examples}

In this section, I propose how the features will support the design behaviors in practice by presenting hypothetical examples of their use.

As an example of how these features support design behavior 1, examine Figure 1. Shown is a sketch that includes representations UML class diagrams, user interface mockup, and lists of requirements. Scraps \& connectors provide the building blocks to compose class diagrams and relate them to one another, and stacking scraps on top of one another allows one to represent containment within a class diagram. The user interface mockups are also sketched on top of scraps, where parts of the user interface element can be repositioned because they are scraps.

To show design behavior 2 as performed with scraps and connectors, I remodel the example from Figure X in chapter Y (from background for instance) in Calico.  Figure 2

For the second design behavior, designers will likely use scraps with little content, while occasionally creating very detailed scraps, because they will only add detail when they need it. Figure X presents an example of a set of scraps representing a set of software components, but only one scrap, ``Mobile vehicle'', has additional details inside of it. Further, designers will only add notational detail when they need it, such as the cardinality of ``5'' present next to ``Mobile vehicle''.

Design behavior 3, they refine and evolve their sketches over time, is supported by scraps and connectors because they enable plain sketches to be refined into scraps in a step-wise fashion. The example in Figure 3, for instance, shows how a list of components can be refined into a UML class diagram. In Figure 3b, the list of components first have a box that is hand drawn around each of them, not yet utilizing scraps. The components in Figure 3c are converted into scraps using the press-and-hold gesture. In Figure 3d, the components are arranged, related, and stack on one another to show containment. 

Design behavior 4, they use impromptu notations, is supported by scraps and the palette because they allow the designer to save and reuse visual icons to the palette. Figure 4, for instance, includes specialized tags and icons, such as the ``car'' field, which is dropped on top of other scraps in the canvas to signify that those other classes contain the car field. The palette holds common elements and fields, enabling the designer to rapidly reuse those elements without needing to recreate them each time the designer needs them.

For the fifth design behavior, moving between perspectives, designers use a combination of scraps and intentional interfaces in order to move between perspectives. Within a canvas, they use scraps to create representations of different types, where scraps help them to move those representations out of the way in order to create more space for additional representations. Between canvases, intentional interfaces aid designers in moving between perspectives created on different canvases by using the cluster view, the back and forth navigation buttons, and the breadcrumb bar.

For the sixth design behavior, designers use scraps, the palette, and intentional interfaces in order to move between alternatives. Within a canvas, scraps help designers generate and move between alternatives by using scrap copy to generate alternative versions of a scrap quickly. Between canvases, designers use the palette to transfer scraps, or to copy a canvas in order to generate an alternative of an entire canvas. Using tags, designers specifically tag the new canvas as an alternative.

For the seventh design behavior, designers use both scraps and intentional interfaces in order to help them in moving from one level of abstraction. With scraps, they move to a deeper level of abstraction deeper by creating a copy of scrap and then enlarging it make space for more details, or by creating a larger scrap around the existing scrap, as in Figure X. With intentional interfaces, they use multiple canvases to step through different levels of detail in the software architecture of a large system. Given that the total space necessary to sketch the architecture of a system spans more than one canvas, depicting pieces across multiple canvases helps in exploring the parts that the designers find interesting. Those canvases are further explicitly tagged as a level of abstraction.

For the eight design behavior, designers use the fading highlighter in order to mentally simulate over a sketch. As depicted in Figure X, they trace their finger or pen across the sketch without leaving marks, helping them think through the problem.

For the ninth design behavior, scraps and intentional interfaces aid designers in juxtaposing sketches. Using scraps, they rearrange sketches within a canvas so that the sketches are juxtaposed against one another. Further, scraps make sketches larger to better compare them, or smaller to create empty space for juxtaposing other sketches. Using intentional interfaces, designers zoom in to canvases in the cluster view, and further move canvases next to one another in the cluster view to compare and reference them, as depicted in Figure X.

For the tenth design behavior, they designers use intentional interfaces to review their progress. From the cluster view, designers arrangement their canvases so that related canvases are next to one another, which helps them in reviewing work that spans multiple canvases. The tags on canvases further reminds the designers of their intent when creating canvases, for example if the canvases represented alternatives, levels of abstraction, or different perspectives. Also, the history navigation buttons in the canvas, which allow designers to navigate backwards and forwards through the set of visited canvases, help them trace their steps.

For the eleventh design behavior, they use intentional interfaces to retreat to previous ideas. Intentional interfaces supports this design behavior by allowing the design to create as many canvases as they need, and relieve them of needing to erase past work. Intentional interfaces further provides several levels of grouping, such as grouping canvases into clusters, and by linking canvases using tags. These methods of organization help the designer in navigating back to old content.

For the twelfth design behavior, they use intentional interfaces to support them in switching between synchronous and asynchronous work. By working in the same canvas, designers work together synchronously on a design, in which the strokes made by one designer is immediately visible by the other design occupying the same canvas. If the second designer is stricken with inspiration for a new idea, that person copies the contents of the current canvas into another canvas, where they explore the new idea asynchronously.

For the thirteenth design behavior, designers use the fading highlighter to explain their sketches to one another. When two or more designers are occupying the same canvas, the first designer uses the fading highlighter to trace over their sketch, which the other designer immediately sees on their own canvas. With this feature, designers explain their sketches by circling or underlining parts of their sketch for emphasis, or by tracing their finger along box-and-arrow diagrams to demonstrate the flow of data.

For the fourteenth design behavior, designers use intentional interfaces and the palette to bring their work together. With intentional interfaces, they copy the contents of their canvas, and refer to the contents of their previous canvas in order to merge the content. With the palette, they merge their content by copying scraps from several canvases onto a single canvas, and generate a new design on the final canvas..

These examples do not represent the full range of expected behavior, but they provide a high level overview of how the features in Table \ref{table:calico-version-two:designbehaviors} will support their respective behaviors.

\section{Implementation Notes}

Calico Version Two's implementation consists of approximately 100,000 lines of code written in Java 7.0. Calico was developed using the Eclipse development environment, and was tested on Hitachi FX-Duo Starboard interactive whiteboards. It is portable across several operating systems (i.e., Windows XP/Vista, Linux, and Mac OSX), though all of my laboratory evaluations described below were performed using a Starboard connected to a Windows machine that ran Calico. ASUS EEE121 tablets were also used.

\subsection{Architecture}

The architecture of Calico Version Two is based on a client-server architecture. A single server instance manages the content of all canvases, all clients connect to the server transmit inputs from their respective user, and the server broadcasts all updates from clients to all other connected clients. The server is strictly a headless application that listens for clients on a pre-determined port, and establishes a synchronous TCP connection all connected clients. In order to reduce the amount of network bandwidth used during usage, Calico Version Two mostly (but not strictly) uses smart clients, in which much of the computational work in Calico, such as geometry for resizing and rotating scraps,is done on client machines and the results are sent to the server. In our own usage, a single Calico server instance is capable of handling up to 20 users sketching simultaneously before performance is severely impacted. The aforementioned scenario involved usage by students in a classroom scenario.

In order to manipulate the state of the canvas, Calico Version Two uses a command pattern. An individual command consists of a byte array, which is formally called a CalicoPacket within the architecture. The CalicoPacket is Calico's own binary format for transmitting information, and is more compact than other more verbose messaging protocols such as XML and JSON. Both the client and the server contain a matching master list of all possible commands called NetworkCommand class. In order to perform a command, such as a user drawing a pen stroke, a CalicoPacket is created that contains the pen stroke command, the command is interpreted by a singleton command interpretation class, which converts the CalicoPacket from raw byte array data to usable values, and performs the command. The CalicoPacket is used to send and receive information from the server.

The client uses an architecture that is inspired by the MVC design pattern, with some slight variations. The client interface is built on top of the Piccolo2D architecture. In this framework, we build on the construct \textit{PCanvas} to represent the container of all elements, and \textit{PNode} to represent all objects on a PCanvas. In our architecture, both the PCanvas and PNode classes serve the purpose of both the model and the view. There is a further set of controllers that manage manipulating the model, and painting the representation of the model to the view. Each model element may always be represented as a CalicoPacket. 

The core set of classes that form the content in Calico are \textit{CCanvas}, \textit{CStroke}, \textit{CScrap}, and \textit{CConnector}. The container for all sketches is represented by CCanvas, which itself inherits from PCanvas. A CCanvas may contain any sketch element, which includes the set of elements that extend PNodes. These elements that extend PNode are CStroke, CScrap, and CConnector. A Scrap may contain both strokes and other scraps. A connector has a one-to-two relation with scraps, in which the head of a connector is associated to a scrap and the tail another.

In order to manage input from the pen, a visitor design pattern is used to accommodate a flexible amount of modes (drawing, erasing, and highlighting), with a flexible number of unique objects. The interface for all objects that are traversed in the visitor pattern include: \textit{actionPressed}, \textit{actionDragged}, and \textit{actionReleased}. When receiving input from the pen, an input controller first processes the input to determine the appropriate handler that implements the traverser pattern, then forwards the event to that handler. Both the canvas and the scrap implement the visitor design pattern to uniquely handle input. When the visitor design pattern visits the canvas handler, a stroke is created on the canvas. When it visits a scrap, the stroke is drawn directly to the scrap. A benefit of using a visitor pattern for handling input is that future plugins may provide new input handlers to Calico. For example, the scrap-list was first added as a plugin and uses a custom handler for organizing elements.

The architecture further contains a plugin design pattern supported by an event handler and event dispatcher. The motivation behind the plugin architecture of Calico was to partition developer contributions to Calico's codebase. By modularizing Calico's components, several individuals could more easily contribute code to Calico without creating conflicts each others' contributions. Both the palette and intentional interfaces are developed as plugins to the system. In order for a plugin to extend the system, a plugin may register itself as a listener to specific commands. In turn, when a command is performed by the command interpreter, the event handler broadcasts the event to all plugins, which may choose to operate on that action.

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 
 \newpage \chapter{Evaluation}
\label{chapter:evaluation}

In the previous chapter, I presented the final version of Calico with features that address all fourteen of the design behaviors, as well as examples of how the features may support the design behaviors in practice. In this chapter, I present an evaluation of Calico Version Two's ability to support those design behaviors through an ``in the field'' qualitative evaluation. In this chapter, I simply report the experiences from those who used Calico Version Two in the course of their own work and map their use onto the design behaviors. In Chapter \ref{chapter:discussion}, I reflect on these experiences.

Within the evaluations of Chapters \ref{chapter:calico-version-one}, I presented the study of software designers in a controlled setting, designing a solution for a hypothetical problem. This setting provided the evidence to suggest that scraps, the palette, and the grid can indeed support a subset of the fourteen design behaviors. However, in moving forward to evaluate the final version of Calico, it is prudent to consider the shortcomings of an evaluation conducted within a controlled environment. First, it is possible that the short duration of the laboratory studies does not give participants sufficient time to become accustomed to Calico's features. Longer-term use will allow groups to receive more practice, and learn how to use the features more effectively. Second, group behaviors such as switching between synchronous and asynchronous work and explaining their sketches are more pronounced in groups of three or more. Design behaviors based on group activities do indeed happen with only two designers, but the support Calico provides may have a more noticeable impact on collaborative work in larger groups than two. Third, Calico may have other benefits for group projects that may not appear in our analysis framework. For instance, Calico may help group meetings start more quickly because the sketches persist between one meeting and another.

Therefore, in this chapter, I report on a longer-term, qualitative study of Calico in use at software companies to not only evaluate its support for the fourteen design behaviors, but also understand its role within the context of a real-world design environment. In order to conduct this evaluation, I performed a multi-week qualitative study at three sites, including a commercial open source software company, an interaction design company, and a distributed research group. In this chapter, I report on my findings of how Calico is used over the long term at these three settings, how users incorporated Calico into their own work, and how it affected the way they design.

After reporting on each site, I report on the design behaviors as they occurred collectively across all three sites. My focus is to examine if the design behaviors occurred  while using Calico, and if so, what features supported those behaviors beyond how they would manifest themselves on a regular whiteboard.

The rest of this chapter is organized as follows: Section \ref{chapter:evaluation:overview} describes the setup of the study, the participants, and the data collections used. Sections \ref{chapter:evaluation:deployment1} - \ref{chapter:evaluation:deployment3} presents the results of the field studies framed within the research questions proposed put forward in Section \ref{chapter:evaluation:overview} and Chapter \ref{chapter:research-question}. More specifically, Section \ref{chapter:evaluation:deployment1} describes the findings at the commercial open source software company. Section \ref{chapter:evaluation:deployment2} describes the findings at the interaction design software company, and Section \ref{chapter:evaluation:deployment3} describes the findings with the distributed research group. Section \ref{chapter:evaluation:design-behaviors} collectively reviews the three field sites for evidence of design behaviors and how they were supported by Calico's features. 

\section{Method}
\label{chapter:evaluation:overview}

I evaluated Calico at three field sites, including two commercial companies and a geographically-distributed research group. At each site, I verified that a Calico installation was setup correctly, which included a large electronic whiteboard, a server instance of Calico, and access to pen-based tablets that could also access Calico. All participants were trained in the use of Calico.

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=8cm,keepaspectratio]{./figures/Evaluation/setting-opensource}
  \caption{The physical setup at the commercial OSS company.}
  \label{fig:evaluation:setting-opensource}
\end{figure*}

The first group is a commercial open-source software company, located in southern California, that develops software within the healthcare industry. In this text, I collectively refer to the members of this company as the OSS group. A software team of five people within the company, including four software developers and one team lead, used Calico in their project. In this deployment, depicted in Figure \ref{fig:evaluation:setting-opensource}, a Hitachi Starboard FX was installed on site, as well as three ASUS EEE121 tablets for use with the whiteboard. Within the same space, there were two couches present in front of the board, and a regular whiteboard next to the couches. All group members were given a tutorial on how to use Calico Version Two, and shown how to launch Calico and connect to the server from their own machine.  For the duration of the study, the team was engaged in architecting and implementing the next version of their project, which was written using Java software. The board itself was physically adjacent to their desks. This group was evaluated over a four week period.

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/Evaluation/setting-interactiondesign}
  \caption{A tutorial of the usage of Calico given to the interaction design group.}
  \label{fig:evaluation:setting-interactiondesign}
\end{figure*}

The second group was an interaction design firm located in northern California. In this text, I collectively refer to the users from this company as the interaction design group. In this deployment, depicted in Figure \ref{fig:evaluation:setting-interactiondesign}, a Hitachi Starboard FXDUO88 was installed on site. Upon installation, the members of the company were invited to a tutorial and shown how to launch Calico from their own machines and connect to the boards. While no tablets were installed, as was done with the OSS group, the company did already have pen-based tablets which could launch Calico. Within the company, two individuals used Calico to support their work. In contrast to the OSS group, the individuals at the interaction design company were not responsible for code, but instead used Calico to perform interaction design activities, such as creating user personas using notes from interviews and creating storyboards. The pair of individuals used Calico over a five day period to process their interviews.

\begin{figure*}[tbh]
  \centering
  \includegraphics[width=16cm,keepaspectratio]{./figures/Evaluation/setting-researchgroup}
  \caption{The physical setup of the research group.}
  \label{fig:evaluation:setting-researchgroup}
\end{figure*}

The third group was a distributed software research group located at the University of California, Irvine on the west coast and Carnegie Mellon University on the east coast. In this text, I refer to this group as the research group. This deployment, depicted in Figure \ref{fig:evaluation:setting-researchgroup}, mixed existing users of Calico at UC Irvine, and people new to Calico at CMU. Two members were located on the west coast and had access to two adjacent Hitachi Starboard FXDUO77 machines, as well as tablet based machines. One member was located on the east coast with access to an HP tablet machine with an electronic pen. This research group collaborated on a research project for a massive online development environment for developing Javascript projects. This group was tasked with designing the front end and writing the software for their project. While they used Calico for a long period of time, only the most recent seven months are evaluated. During that time, the CMU member visited UCI for a week of intense work with Calico; though it saw frequent use thereafter. While apart, skype was used to coordinate their weekly meetings, and was used alongside Calico.

\subsection{Data collection}

The observations presented in this chapter are primarily based on data collected from interviews and usage logs generated by Calico. The evaluation period for each site was at least four weeks, and the participants at each site were free to use the system as much, or as little, as desired. Given the long duration of the study, and that usage of Calico by users at each site was opportunistic rather than planned, we did not use video.

In order to review the design activity performed using Calico, I examined the usage logs produced by Calico. Each action taken by users was recorded into a history file, including all drawing activity, navigation, usage of features, and accessing content from remote machines. Each user action included a timestamp, name of the machine that was running the Calico client, action performed, and an image of the canvases at the time the action was performed. 

Based on examinations of logs and the content drawn, I conducted semi-structured face-to-face interviews. I conduct two semi-structured interview with the OSS group, one with the interaction design group, and one with the research group. In each interview, I began by asking ``What was your most vivid design experience with Calico?'' From this question, participants typically walked through the designs they created in Calico, physically pointing, mimicking their usage of features, and describing the contents of, as well as the activity leading to, their designs. During these walkthroughs, I asked for clarification of content within their designs as well as activity that I observed from their usage logs.

In subsequent questions, I specifically targeted the first two research questions. First, in order to evaluate whether Calico could be considered ``minimally invasive'', I asked participants if they needed to make any compromises in using Calico, what obstacles or surprises they encountered in creating their designs and if they struggled with the features. I also inquired how design sessions would have gone had they not used Calico. 

Second, in order to evaluate if Calico has a ``coherent set of features'', I asked participants to report their experience on using the features in general, what features they found helpful, and if the features clashed with one another. 

Due to the sensitive nature of some of the data collected, measures were taken to ensure that data was securely collected. Of the three sites visited, those in charge of two of the sites (the OSS group and the research group) provided consent to use the data collected for academic publication. The third site (the interaction design group), however,  was under a strict NDA agreement, and all images, logs, and raw data were not permitted to be taken off-site. In this case, usage logs were processed and reviewed on-site, and only personal notes were permitted to be taken off-site. Furthermore, for the purpose of publication, relevant images containing sensitive content were redrawn such that key insights, such as usage of notations, are preserved, but the content itself is anonymized.

\subsection{Data analysis}

Based on the logs and interviews, collected data from each session was reviewed in two phases. First, I reviewed interviews for each group for responses pertaining to the overall design activity that occurred, their general feedback on their experiences using Calico, and feedback on specific feature usage. In the second phase, I reviewed both the usage logs and interviews and categorized my observations into the following three categories: (1) how they navigated between canvases, (2) the representations they used, and (3) how they conducted group work (mimicking the three categories of design behaviors). 

%Lastly, the interview notes and logs were examined for any surprises that do not fit within the research question posed.
The results from the three field sites are partitioned across Sections \ref{chapter:evaluation:deployment1}, \ref{chapter:evaluation:deployment2}, and \ref{chapter:evaluation:deployment3} due to the variety of behaviors observed. The observations across all three sites are aggregated in the overarching discussion presented in Chapter \ref{chapter:discussion}.

\section{Results: Deployment at a commercial open source software company}
\label{chapter:evaluation:deployment1}

Of the entire duration that Calico was installed at the OSS group, the developers reported using Calico in three extensive design sessions. Across all three sessions, the developers were engaged in developing the next version of a healthcare ``message processing'' tool. 

\subsection{Overview of design sessions}

In the first design session, depicted with excerpts of the canvases in Figure \ref{fig:ossgroup:session1}, the developers turned to the whiteboard to sketch out a user interface for creating custom handlers for different types of messages. In the second design session, depicted with excerpts of the canvases in Figure \ref{fig:ossgroup:session2}, the developers turned to the whiteboard to design a set of slides that will be used to explain the software architecture of their system. In this case, all members already understood the architecture, but wanted to create a representation ``that was easy to understand''. In the third design session, depicted in Figure \ref{fig:ossgroup:session3} with excerpts, while coding, a developer turned to Calico in order to refactor source code that needed to be updated for their upcoming software release.

In the following, I group my observations of the developers' usage of Calico based on how they navigated between canvases, the representations they created, and how they used Calico to manage group work.

\begin{figure}%
  \centering
  \subfigure[Exploration of entities using box-and-arrow diagrams] {
      \label{fig:ossgroup:session1:a}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session1/canvas1}
   }
  \subfigure[Hierarchical perspective of component structures from Figure \ref{fig:ossgroup:session1:a}] {
      \label{fig:ossgroup:session1:b}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session1/canvas2}
   }
  \subfigure[Refined mock-up for user interface in Figure \ref{fig:ossgroup:session1:a}] {
      \label{fig:ossgroup:session1:c}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session1/canvas3}
   }
  \subfigure[Alternative user interface mockup to Figure \ref{fig:ossgroup:session1:c}] {
      \label{fig:ossgroup:session1:d}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session1/canvas4}
   }         
  \subfigure[Mock-up of use cases created after discussion of Figure \ref{fig:ossgroup:session1:d}] {
      \label{fig:ossgroup:session1:e}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session1/canvas5}
   }   
   \caption {Representations used by the OSS group in their first design session to design a user interface.}
   \label{fig:ossgroup:session1}
\end{figure}%

\begin{figure}%
  \centering
  \subfigure[Minimal sketch of software architecture for ``Messaging Processing'' and ``Event Bus''] {
      \label{fig:ossgroup:session2:a}     
      \includegraphics[width=6.8cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session2/canvas1}
   }
  \subfigure[Less abstracted perspective of Figure \ref{fig:ossgroup:session2:a}] {
      \label{fig:ossgroup:session2:b}     
      \includegraphics[width=6.8cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session2/canvas2}
   }
  \subfigure[Different perspective of ``Event Bus'' in Figure \ref{fig:ossgroup:session2:b}] {
      \label{fig:ossgroup:session2:c}     
      \includegraphics[width=6.8cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session2/canvas3}
   }
  \subfigure[Lower level of abstraction of Figure \ref{fig:ossgroup:session2:c}, ``Alert'' is a child class of ``Event Listener''] {
      \label{fig:ossgroup:session2:d}     
      \includegraphics[width=6.8cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session2/canvas4}
   }
\subfigure[Different perspective of Figure \ref{fig:ossgroup:session2:a}, Software architecture of the ``Alert'' event listener] {
      \label{fig:ossgroup:session2:e}     
      \includegraphics[width=6.8cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session2/canvas5}
   }
  \subfigure[Lower level of abstraction of ``Action Group'' in Figure \ref{fig:ossgroup:session2:e}] {
      \label{fig:ossgroup:session2:f}     
      \includegraphics[width=6.8cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session2/canvas6}
   }
  \subfigure[Different perspective of Figure \ref{fig:ossgroup:session2:e}, components summarized in list] {
      \label{fig:ossgroup:session2:g}     
      \includegraphics[width=6.8cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session2/canvas7}
   }   
  \subfigure[Use case diagram encompassing Figures \ref{fig:ossgroup:session2:a} to \ref{fig:ossgroup:session2:g}] {
      \label{fig:ossgroup:session2:h}     
      \includegraphics[width=6.8cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session2/canvas8}
   }   
   \caption {Representations used by the OSS group in their second design session in which they worked on representation that details the software architecture for handling different types of messages.}
   \label{fig:ossgroup:session2}
\end{figure}%

\begin{figure}%
  \centering
  \subfigure[First iteration of source code] {
      \label{fig:ossgroup:session3:a}     
      \includegraphics[width=14cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session3/canvas1}
   }
  \subfigure[Second iteration of source code] {
      \label{fig:ossgroup:session3:b}     
      \includegraphics[width=14cm,keepaspectratio]{./figures/Evaluation/ossgroup/Session3/canvas2}
   }
   \caption {Representations used by the OSS group developer to refactor their code in the third reported design session.}
   \label{fig:ossgroup:session3}   
\end{figure}%

\subsection{Feature usage}
\label{chapter:evaluation:deployment1:part1}

\subsubsection{General feedback}

On the whole, the individuals that did use Calico reported that they did not find the features to be invasive to the basic sketching activity. At a bare minimum, all individuals said they were able to use the large electronic whiteboard as they would a regular whiteboard. Individuals reported some drawbacks with the hardware itself, in which the electronic whiteboard hardware experienced lag so they needed to write slowly to create legible representations. However, the pen-based tablets were much more precise. They also reported some surprises in the usability of the tool, for example they expected the forward and backward navigation buttons to step through the chain of linked canvases in the intentional interface, instead of the visited canvases. Also, the members of the OSS group expected greater functionality in changing colors of scraps and text-scraps, but fell back to plain sketching to use colors.

When asked if they made any compromises in order to perform their activities using Calico, users responded that they did not need to make many compromises. The only compromises that they reported were writing slower and larger on the electronic whiteboard. If they did not use Calico, they reported that they would have moved to a meeting room with several whiteboards rather than the space immediately adjacent to their desks. One participant reported that in one activity they would have written on the window using erasable markers, but instead chose to use Calico. They also reported that they normally would have taken pictures of their work, but instead used the email feature of Calico.

\subsubsection{Feature specific}

When asked about the features as a whole, the members of the OSS group reported that they did not feel that the features conflicted with one another. Additionally, both interviews and usage logs confirmed that all features were indeed used at least to some extent, with the exception of the palette, which was very rarely used. The following captures the general feedback from the OSS group concerning each feature.

\textbf{Basic sketching and features.} Aside from the issues with the responsiveness of the large electronic whiteboard mentioned above, OSS group members reported that the basic sketching features were (e.g., changing pen color, changing line width, undo, and redo) helpful. Sketching on regular tablets was responsive, and developers from the OSS group stated that synchronous sketching was helpful in meetings, allowing members to work together, or hand tablets back and forth between each other while designing with the large electronic whiteboard. They reported frequent usage of multiple colors and the undo/redo functionality.  They did, however, mistake the ``clear canvas'' button for the ``eraser mode'' button several times, leading to the canvas being accidentally cleared, but this action could be undone with the ``undo'' functionality. Two of the developers in the OSS group reported some difficulty adapting because upon using the device they default to gestures they use on other devices such as pinch-and-zoom, which are not available in Calico. Another member of the OSS group requested additional colors be made available as well.

\textbf{Scraps.} The participants found the use of scraps helpful on occasion. One member from the OSS group wanted to create text-scraps with different colored text, but instead turned to writing by hand instead of typing when this was not possible. Further, when using text scraps at the large electronic whiteboard, members from the OSS group found it cumbersome to switch between the bluetooth keyboard and the large display, and stated that they would have preferred an on-screen keyboard. Writing content was sloppy, so where possible, they created text scraps to make content more legible. Otherwise, OSS group members reported that scraps were sufficiently helpful to move content and represent objects. Among the three design sessions observed, scraps were used to represent objects in two of those sessions, and in the other session, were used simply to manipulate plain sketched content.  In another design session, one of the members of the OSS group encountered difficulty in using the ``shrink-to-contents'' functionality for scraps, which creates a rectangular scrap around sketches if present, or simply a rectangle if no content was present. The OSS group member reported wanting to create a rectangle, but the ``shrink-to-contents'' button instead created a smaller rectangle around the existing sketch. Lastly, a member of the OSS group requested that scraps be resized with a corner anchor, without locking their aspect ratio.

\textbf{Palette.} The palette received light usage by the OSS group. The palette was used in one of the three design sessions, as a few scraps were used across several canvases. When asked how often they reused scraps, the OSS group collectively replied ``rarely''. They reported that they typically went to a new blank canvas when discussing their software system, after which they archived the canvases used by emailing them to the entire group. With respect to reusing scraps with the palette, they stated that it was faster to simply redraw their representations.

\textbf{Intentional interfaces.} OSS group members responded that they found the ``chaining'', i.e., linking canvases using tags, as supported by intentional interfaces to be helpful in capturing individual design sessions. They used the ``new canvas'' and ``copy canvas'' buttons several times with tagging to create chains, and all design activities were done in a single cluster. When drawing within a canvas, one OSS group member reported that they exclusively used the breadcrumb bar to navigate between canvases. Another developer from the OSS group preferred to use the cluster view, but reported being annoyed that the cluster view changed their zoom perspective in an unexpected ways. In their words, the view ``kept jumping around''. As a result, the users reported a lot of unnecessary panning in the cluster view. Two other developers in the OSS group also reported that they found the forward and backward navigation buttons confusing. They expected these buttons to navigate to the next canvas in the chain within the intentional interface, not the most recently visited canvas. One developer in the OSS group asked for a minimap of the cluster view to be directly visible in the canvas. One member of the OSS group called the organization ``wizardry'', since they could not see how it changed.

\textbf{Fading highlighter.} The OSS group reported that the fading highlighter was useful within group sessions. They used the fading highlighter in two of the three sessions, and reported that they used this feature to explain designs.

\subsection{Canvas navigation}

\textbf{Using intentional interface chains to capture design sessions.} The OSS group reported that they organized each design session by ``chaining'' canvases within the cluster view, as depicted in Figure \ref{fig:ossgroup:clusterview}. Each design session was linked together using tags, typically with either the ``perspective'' tag or the ``alternative'' tag. Each new design session began as a new canvas within the center of the radial circle, and extended outward using canvases linked as chains. A member of the OSS group reported that panning through their chains of canvases was helpful in reviewing the sketches that they created.

\begin{figure}%
  \centering
  \includegraphics[width=12cm,keepaspectratio]{./figures/Evaluation/ossgroup/ossgroup-clusterview}
   \caption {Cluster view of canvases. The three major sessions were grouped by ``chaining'' canvases.}
   \label{fig:ossgroup:clusterview}   
\end{figure}%

\textbf{Using canvases to explore perspectives.} The OSS group reported that they used several canvases to explore their designs across multiple perspectives. They explored different perspectives both by using different types of representations, such as user interface mockups and software architecture diagrams, and also by creating sketches that depicted different parts of the system. 

In their first design session, depicted in Figure \ref{fig:ossgroup:session1}, they created sketches using different types of representations to explore the components of the user interface and its behavior at runtime. In Figure \ref{fig:ossgroup:session1:a}, they used box-and-arrow diagrams alongside low-detailed user interface sketches to brainstorm major components of the user interface and their relationships to one another. In Figure \ref{fig:ossgroup:session1:b}, they shifted to a hierarchical list to record the structure. In Figures \ref{fig:ossgroup:session1:c} and \ref{fig:ossgroup:session1:d} they depicted the contents of Figure \ref{fig:ossgroup:session1:b} as it would appear to the end user. In Figure \ref{fig:ossgroup:session1:e}, they sketched out three different use cases, where ``CH'', ``CH2'', and ``CH3'' represent three different results for input entered by the user in the scrap ``All Channels''.

In their second design session, depicted in Figure \ref{fig:ossgroup:session2}, they explored different parts of the same design across the different canvases in order to document the flow of information through the system. For example, Figure \ref{fig:ossgroup:session2:a} shows ``message processing'' sending information to the ``Event Bus''. Figure \ref{fig:ossgroup:session2:c} shows how information is passed from the ``Event Bus'' scrap onto the ``Event Listener'' scraps. Figure \ref{fig:ossgroup:session2:d} further shows how information is passed from the ``Alert'' scrap, which itself is an instantiation of an ``Event Listener'', onto the ``Email'' and ``Channel'' scraps. The OSS group members sketched out each step in order to thoroughly explain the rules and logic used for passing information between components. 

\textbf{Stepping through a design using levels of abstraction. } The OSS group used multiple levels of abstraction to aid in explaining their architecture. They used it both to progressively step into the detail of their design, and also to back away to once again see the bigger picture.

When stepping progressively into their design, they made use of the ``copy canvas'' feature to repeat a canvas, but replace parts of their sketches with more detailed versions to step into lower levels of abstraction. The second design session in Figure \ref{fig:ossgroup:session2} shows two such cases. First, Figure \ref{fig:ossgroup:session2:b} shows a lower level of abstraction of Figure \ref{fig:ossgroup:session2:a}. Between Figures \ref{fig:ossgroup:session2:a} and \ref{fig:ossgroup:session2:b}, ``message processing'' is replaced by scraps labeled as ``Channel'' , which represent the components that process the messages.  In Figure \ref{fig:ossgroup:session2:b}, each colored connector represents messages of a specific type that are passed from a ``Channel'' to the ``Event Bus''. Second,  Figures \ref{fig:ossgroup:session2:c}, \ref{fig:ossgroup:session2:d}, \ref{fig:ossgroup:session2:e}, and \ref{fig:ossgroup:session2:f} show a sequential step-through in levels of abstraction. The ``Event Listener'' scrap in Figure \ref{fig:ossgroup:session2:c} is represented as the ``Alert'' scrap in Figure \ref{fig:ossgroup:session2:d}, where it is shown in much greater detail. The ``AG1'' scrap in Figure \ref{fig:ossgroup:session2:d} is represented as the ``Action Group'' scrap in Figure \ref{fig:ossgroup:session2:e}, where it is elaborated in much more detail. The ``Action Group'' scrap in Figure \ref{fig:ossgroup:session2:e} is again represented in ``Action Group'' in Figure \ref{fig:ossgroup:session2:f}, showing further sub components for ``Action Group''. While creating these scraps, the OSS group used the palette to copy elements between canvases, such as ``Event Bus'' and ``Action Group''.

After their stepwise exploration leading to Figure \ref{fig:ossgroup:session2:f}, the OSS group members sketched a high level picture of messages passing through the system in Figure \ref{fig:ossgroup:session2:h}. Not shown in the figure, the OSS group members created several copies of the contents of Figure \ref{fig:ossgroup:session2:h} in order to explain how different types of messages may pass through the system. In their session, they returned to previous canvases when they had questions about particular components.

\textbf{Copying canvases to generate alteratives. } The OSS group sometimes used multiple canvases to create alternative solutions to existing designs. For example, a member of the OSS group reported that, while working with their team members on the user interface in Figure \ref{fig:ossgroup:session1:c}, they ``did not quite agree with the design'' and deviated from the group to create their own version. To do so, they used the ``copy canvas'' button, selected the ``alternative'' tag in the intentional interface tag panel, and created a new user interface, the result of which is shown in  \ref{fig:ossgroup:session1:d}. 

In a third design session, one of the OSS group developers used multiple canvases to help them in iterating on past designs. After finishing the design in Figure \ref{fig:ossgroup:session3:a}, the OSS group member actually implemented the designed changes in the software, to later return to the large electronic whiteboard to verify that their source code correctly implemented the original pseudocode in Calico. In this second iteration, they created a copy of the original canvas and pasted a screenshot of the source code that resulted from the previous design session, where they continued to refine the pseudocode.

\subsection{Representations}

The OSS group created several types of representations in their design sessions.

\textbf{Box-and-arrow diagrams. } Across the design sessions, they created what they termed ``box-and-arrow'' diagrams. They did not make any direct reference to notations such as UML class diagrams or ER notation, but rather viewed their sketches as rudimentary abstractions of their existing software system. The first was used for brainstorming, and the second for explaining an architecture.

In the first design session, depicted in \ref{fig:ossgroup:session1:a}, they created a set of box-and-arrow diagrams to brainstorm the components of a user interface. The design session began by first listing out the relevant entities in the software, which were the text scraps ``Source'' and ``Destinations''. In this session, the OSS group experimented with different combinations of actions, which were reflected with further text scraps associated with connectors at the bottom. They used connectors to represent the data sources that the `Source'' and ``Destinations'' scraps could pull from and send to.

In the second design session, the OSS group used a combination of scraps, connectors, and plain sketches to represent software components and the passing of data. When asked why they sometimes represented components as plain sketches and other times using scraps, they reported that they used scraps to represent components that they were actively designing and using. The OSS group reported that elements in sketches, such as ``In'' and ``Out'' in Figure \ref{fig:ossgroup:session2:a} and LLP in \ref{fig:ossgroup:session2:h}, represented outside information. Creating items such as ``Event Listener'' in Figure \ref{fig:ossgroup:session2:c} allowed the components to be reused and moved. For passing data, they simply used colors to represent particular types of data that were passed. For example, between Figures \ref{fig:ossgroup:session2:a} and \ref{fig:ossgroup:session2:b}, the OSS group used four different types of colors to show the types of data that are passed from a ``Channel'' to the ``Event Bus''. In Figure \ref{fig:ossgroup:session2:c}, the OSS group repeated the use of colors to show how the different types of data were handled.

\textbf{User Interface Mockups. } Across the sketches in Figure \ref{fig:ossgroup:session1}, the OSS group used a mix of plain sketching with scraps to design a component and its interface across several canvases. In both Figure \ref{fig:ossgroup:session1:a} and \ref{fig:ossgroup:session1:b}, they sketched the software entities alongside a user interface mockup. In Figure \ref{fig:ossgroup:session1:a}, the OSS group reported that they created three low-detailed user interface mockups while ``throwing ideas out on the board''. They iterated over their user interface across multiple canvases, two of which are depicted in Figures \ref{fig:ossgroup:session1:c} and \ref{fig:ossgroup:session1:b}.

%- While developing these, they sketched a low-detailed user interface within a scrap at the top of Figure \ref{fig:ossgroup:session1:a}, in which the ``plus'' symbols can be toggled by the user and the resulting configuration is displayed to its immediate right. The right side of Figure \ref{fig:ossgroup:session1:a} represents a quick draft of the user interface proposed in the top left. In Figure \ref{fig:ossgroup:db1b}, the developer created another perspective of the elements in Figure \ref{fig:ossgroup:db1a}, but instead organizing the entities using a hierarchy, and drafted another user interface. This was about throwing ideas out on the board.

\textbf{Lists. } The OSS group used lists in two instances. First they used lists as a method to hierarchical organize the user interface in Figure \ref{fig:ossgroup:session1:b}. Second, they used lists as a mechanism to describe the functionality of a software architecture in detail in Figure \ref{fig:ossgroup:session2:g}. The OSS group reported that they used lists to summarize their design sessions and to reflect on designs they recently created. However, they did not create many lists because they found that the board reduced the aesthetic quality of their writing.

\textbf{Use case sketches. } In order to capture the behavior of their designs at runtime, they created primitive use case scenarios. For example, in the final canvas of their first design session, Figure \ref{fig:ossgroup:session1:e}, the OSS group sketched the behavior of the user interface when different channels were checked. This representation depicts the results of three different use cases for the interfaces shown in Figures \ref{fig:ossgroup:session1:c} and \ref{fig:ossgroup:session1:d}. The left scrap in Figure \ref{fig:ossgroup:session1:e} represents the input panel, and the three scraps on the right hand side each represent the result of a different use case, in which either ``Ch 1'', ``Ch 2'', or ``Ch 3'' is tapped on the left scrap. A member of the OSS group created these three previews by drafting an initial scrap on the right hand side, and copying it multiple times, adjusting the values for each respective ``Channel'' selected. 

In the second design session, they created several copies of the contents in Figure \ref{fig:ossgroup:session2:h} to explore different use case scenarios. They used different use case scenarios to walk through input messages of different types and frequency, as well as using different event listeners to handle those messages.

\textbf{Source code. } In the third design session, one of the members of the OSS group used the large electronic whiteboard to help him in refactoring his code. Depicted in Figure \ref{fig:ossgroup:session3}, the OSS group member needed to refactor his code in order to process an XML file that contains new fields in the next version of their software tool. In order to carry out his design session, he opened Calico on his own desktop, copy and pasted a screenshot of the XML code that he needed to process, and continued his design session at the large electronic whiteboard. In his exit interview, he needed ``a space to think freely''. While sketching, he used several custom annotations and colors to depict the process flow of different software components through the XML structure. Black vertical bars were used to represent a call stack, which he drew to help him in understanding how far the callstack goes in a recursive call. The red arrows were used to represent how the call stack moves through the data.

Overall, the OSS group member reported that using different colors helped him see things ``at-a-glance''. He stepped back from his pseudocode several times, experimented with walking through the code in different orders, and used a different color for each walkthrough. The final design of the first iteration is shown in Figure \ref{fig:ossgroup:session3:a}. After implementing the design in the code base, and upon discovery that an issue was not addressed in his first iteration, the OSS group member copied and pasted the latest Java source code to once more step through their design again to address the discovered issue.

\subsection{Collaborative work}

\textbf{Preparing work ahead of time.} The first design session was a shared group session that spanned the greater part of a working day. In the morning, one of the OSS group members took a tablet to their desk and worked out a design, and later presented his design in a group meeting with four other developers in the OSS group, where they continued to work together on the design for several hours afterwards. A subset of the sketches produced in this design session are depicted in Figure \ref{fig:ossgroup:session1}. In this design session, they were designing how to handle data as it is passed through each ``channel''.

\textbf{Explaining designs to the group.} In the second design session, the team members made heavy use of multiple tablets, as well as the fading highlighter. When the group meeting began in the afternoon, the OSS group made heavy use of the fading highlighter to explain the behavior of the system shown in Figure \ref{fig:ossgroup:session2:a}. Having worked out much of the design independently earlier that day, a member of the OSS group used the fading highlighter to explain the state of the design thus far. 

\textbf{Spontaneous asynchronous work.} During the design session, some of the design members would privately move to another cell, create a design, and call other members of the group to visit their cell. For example, while the group discussed the data that was passed from ``Event Bus'' in Figure \ref{fig:ossgroup:session2:d}, one of the OSS group memebers wanted to understand how the ``Alert'' component worked in the greater context, and moved to the canvas depicted in Figure \ref{fig:ossgroup:session2:h} to sketch a much more abstract representation to show how data flowed through it. After sketching it, they called the other designers over, and made frequent movements back and forth between these canvases to compare them.

%The first design session, depicted in Figure \ref{fig:ossgroup:session1}, was performed over two separate days. In this design session, the developer was creating a tool in which an end-user may create several ``channels'' for processing data, and for each channel, a ``source'' is chosen and piped to any number of ``destinations''.  The developer created a set of sketches specifically to help himself plan the component that he was developing.
%
%Across the sketches in Figure \ref{fig:ossgroup:session1}, the developer used a mix of plain sketching with scraps to design a component and its interface across several canvases. Within both Figure \ref{fig:ossgroup:session1:a} and \ref{fig:ossgroup:session1:b}, the developer sketched both the the software entities alongside a user interface mockup. The design session began by first listing out the relevant entities in the software, which were the text-scraps, ``Source'' and ``Destinations''. In this session, the developer experimented with different combinations of actions, which were reflected with further text-scraps associated with connectors at the bottom. While developing these, they sketched a low-detailed user interface within a scrap at the top of Figure \ref{fig:ossgroup:session1:a}, in which the ``plus'' symbols can be toggled by the user and the resulting configuration is displayed to its immediate right. The right side of Figure \ref{fig:ossgroup:session1:a} represents a quick draft of the user interface proposed in the top left. In Figure \ref{fig:ossgroup:db1b}, the developer created another perspective of the elements in Figure \ref{fig:ossgroup:db1a}, but instead organizing the entities using a hierarchy, and drafted another user interface.
%
%The developer used both the copying of scraps and the copying of canvases to mock up iterations of the user interface. The developer created two alternative solutions for the user interface in Figures \ref{fig:ossgroup:session1:c} and \ref{fig:ossgroup:session1:d}, the second of which was created by copying a canvas, and selecting ``alternative'' in the intentional interface tag panel. In the final canvas, Figure \ref{fig:ossgroup:session1:e}, the developer sketched the behavior of the user interface when different channels were checked from the user interfaces in Figure \ref{fig:ossgroup:session1:c} and \ref{fig:ossgroup:session1:d}. The left scrap in Figure \ref{fig:ossgroup:session1:e} represents the input panel, and the three scraps on the right hand side each represent the results of different use cases, in which either ``Ch 1'', ``Ch 2'', or ``Ch 3'' are tapped on the left scrap. The developer created the three previews by drafting an initial scrap on the right hand side, and copying it multiple times, adjusting the values for each respective ``Channel'' selected. 


%The second design session was a shared group session that spanned the greater part of a working day. In the morning, one of the developers took a tablet to their desk and worked out a design, and presented his design in a group meeting with four other developers, where they continued to work together on the design for several hours afterwards. A subset of the sketches produced in this design session are depicted in Figure \ref{fig:ossgroup:session1}. In this design session, they were designing how to handle data as it is passed through each ``channel''.
%
%With respect to how the designer organized the work across canvases, they reported that they used different canvases to represent different perspectives and views onto the same component. In Figures \ref{fig:ossgroup:session2:a} and \ref{fig:ossgroup:session2:b}, the developers created representations that built on the logic of passing data through ``channels'' from the the first design session using Calico, but now handle the data by sending it to an ``Event Bus'' scrap. The developers used the copy feature to create Figure \ref{fig:ossgroup:session2:b}, which represents a more detail view of Figure \ref{fig:ossgroup:session2:a} and was tagged as ``perspective'' in the intention view. The developers further extended their design of the system by adding the ``Event bus'' scrap to the palette, and reused the scrap in Figures \ref{fig:ossgroup:session2:c}, \ref{fig:ossgroup:session2:d}, and \ref{fig:ossgroup:session2:e}. From the canvas in Figure \ref{fig:ossgroup:session2:e}, the developers added the ``Action Group'' scrap to the palette, and copied it to another canvas, where they worked that component in greater detail. After having designed the interactions between the components of the system, they summarized the content in the canvas depicted in Figure \ref{fig:ossgroup:session2:g}. In Figure \ref{fig:ossgroup:session2:h}, the developers sketched an example of data passing through the components from the previous sketches, i.e., the ``Channel 1'' scrap, being processed, and sent to another channel.
%
%With respect to the representations used, the developers used a combination of scraps, connectors, and plain sketches to represent software components and the passing of data. When asked why they sometimes represented components as plain sketches or using scraps, they reported that they used scraps to represent components that they were actively designing and using. The developers reported that elements in sketches, such as ``In'' and ``Out'' in Figure \ref{fig:ossgroup:session2:a} and LLP in \ref{fig:ossgroup:session2:h}, represented outside information. Creating items such as ``Event Listener'' in Figure \ref{fig:ossgroup:session2:c} allowed the components to be reused and moved. For passing data, they simply used colors to represent particular types of data that was passed. For example, between Figures \ref{fig:ossgroup:session2:a} and \ref{fig:ossgroup:session2:b}, the developers used four different types of colors show the types of data that was passed from a ``Channel'' to the ``Event Bus''. In Figure \ref{fig:ossgroup:session2:c}, the developers repeated the use of colors to show how the different types of data were handled. 
%
%With respect to designing with the group, the team members made heavy use of multiple tablets, as well as the fading highlighter. When the group meeting began in the afternoon, the developers made heavy use of the fading highlighter to explain the behavior of the system within Figure \ref{fig:ossgroup:session2:a}. Having independently worked out much of the design independently earlier that day, a developer used the fading highlighter to explain the state of the design thus far. During the design session, some of the design members would privately move to another cell, create a design, and call other members of the group to visit their cell. For example, While the group discussed the data that was passed from ``Event Bus'' in Figure \ref{fig:ossgroup:session2:d}, one of the developers wanted to understand how the ``Alert'' component worked in the greater context, and moved to the canvas depicted in Figure \ref{fig:ossgroup:session2:h} to sketch a much more abstract representation to show how data flowed through it. After sketching it, they called the other designers over, and made frequent movements back and forth between these canvases to compare them.





%For what activities did they turn to the whiteboard?
%
%Used a tablet while.
%
%Developer worked it out for himself first. Used a tablet rather than the large whiteboard. Afterwards, brought in another person to help them.
%
%Multiple people were using the board at the same time for different activities.
%
%User needed to get their thoughts written out. Was going to design the next version of their software. Was attempting to come up with a migration path from the old version to the new version. Needed to work out issues. Found color to be useful, to identify importance. Wanted to look at XML code that was relevant. Pasted code that he was working on into the board.
%
%Took a different strategy in direction. Made an  alternative, made another screenshot of different code. Used breadcrumb bar to navigate between canvases. 
%
%Wanted a text box to be able to draw.
%
%Another user was used to interface from apple devices, wanted pinch and zoom.
%
%While the main group was discussing something, one person branched off and developed a new alternative. When he was ready, he proposed it to the whole group. They could have done the same thing with two whiteboards, but then they would have to move around. Being able to work simulataneously is nice.
%
%One user was interested in having.
%
%More willing to draw anything since they know that they can create a new canvas, delete it, or return to a previous.
%
%icons are confusing.
%
%Used

\section{Results: Deployment at an interaction design company}
\label{chapter:evaluation:deployment2}

Of the entire duration that Calico was in use at the interaction design group, the interaction design group reported that they used the board on occasion for notes, and used the system to conduct one major on-going design activity. Usage logs indicate that the design session took place across five days, where usage went on throughout the work day. The two members of the interaction design group conducted the design together.

In order to preserve confidentiality in this section, the topic and content of the material have been obfuscated, and the images have been reproduced. I use generic references without mention of the target domain, and have recreated the images to align with this hypothetical task, but still represent what they drew.

Also, it is important to note that in this group, during their five day design session, they used an earlier version of Calico Version Two that did not yet include the intentional interfaces feature, but instead included the grid. The interaction design group did, however, use the intentional interface feature later in a one hour design session. All experiences reported for this group should be assumed as using the grid, unless the intentional interface feature is explicitly stated.

\subsubsection{Overview of design session}

The design session that took place involved processing a set of fifty interviews that the interaction design group conducted, and building a set of user personas based on those interviewed. The interaction design group reported that they typically would have processed the interviews by printing the faces of the fifty people interviewed onto note cards, and working with the note cards in a physical space. They reported that they could have simply written names, but found speaking to images of the people they interviewed to be more evocative and effective. In the large open space, they would organize the note cards by physically grouping them together according to emergent categories, writing notes on the back of the note cards, and taking pictures of the groupings. 

With Calico, they saw an opportunity to perform the same activity, but preserve work that would have been lost every time they re-arranged the note cards. Instead, they performed this activity by importing images of the fifty individuals that they interviewed into the palette, and dropped these onto the canvas as image scraps. In Calico, they viewed the work they did as non-destructive and preservative, where a particular arrangement of the image scraps could be copied and reorganized an indefinite amount of times. If inspiration struck them regarding an earlier arrangement, they were always able to return to a previous arrangement of the image scraps and continue with the most recent work later.

\subsection{Feature usage}

\subsubsection{General feedback}

Overall, the participants reported that the work they performed in the system was useful, but ultimately found it difficult to adapt their work habits to the structure of the system. Representing images as scraps provided advantages such as reducing the overhead in creating physical printouts for each person they interviewed, reusing them across several canvases, and allowing them to return to past explorations. The interaction design group, however, reported that they had a culture of using OneNote, a paginated sketch-based application, on pen-based windows machines, which led to a clash between the expected functionality of Calico and Calico's actual functionality. For example, they expected functionality such as OneNote's lasso mode for selecting and moving content, and subsequently found Calico's scrapping functionality confusing and difficult to adopt. Also, they found Calico's Grid and Intentional Interface system of managing canvases not as fluid to use as OneNote's system of vertically scrolling through paginated sketches. Further, they reported that their ability to write legibly was limited by the large electronic Starboard hardware, which resulted in larger content that was sketched more slowly than expected. They would have preferred the ability to rest their wrist on the board to write comfortably. Ultimately, however, they worked around this limitation to perform their design work.

The interaction design group reported that using Calico did indeed cause some changes in how they designed. First, they reported that, while different from OneNote's paginated system, they felt the sensation of a greater amount of free space to sketch, and as a result, created more sketches and content than they would have otherwise. Within each canvas, however, they reported desiring more free space, either through zooming and panning of content, or paginated spaces such as within OneNote. 

Additionally, the interaction design group had several feature requests. They found the use of modal dialogs, such as email confirmation dialogs and text entry, to be disruptive during their design sessions. They asked for highlighting functionality that is more analogous to its real-life counter part, i.e., persistent and shading the area behind a text with color, which they reported as using often for tagging items in their sketches. They further reported the sudden switch between sketching an object, and creating scraps to be disorienting, and requested separate modes for drawing, creating, selecting, and moving scraps. They further found the set of graphical icons for buttons and menus to not be straightforward, and requested additional descriptions.

Despite these issues, they produced twenty one canvases of content. Sixteen of those canvases made significant use of scraps, six of which used made use of multiple list scraps. One canvas even used connectors between scraps.

\subsubsection{Feature specific}

\textbf{Basic sketching and features.} Participants reported some difficulty in accurately sketching with the system. They reported that it was difficult to produce legible handwriting. For example, when attempting to use print, Calico would produce cursive text. Other times, Calico would lag and sometimes lose details of their writing.

\textbf{Scraps.} The participants reported that the functionality that scraps enabled, such as moving sketched contents, was helpful, but reported some difficulty in using scraps. The participants reported that the interaction for moving scraps felt slow, particularly when moving scraps to categorize them in tables or quadrants. Further, the system began to slow down when using a large number of image scraps, thirty or more, within the same canvas. This is partly why they requested separate modes.

It is interesting to note, though, that early versions of Calico Version Two had such separate modes, which was received with very mixed reactions when deployed to students in a software design class. Students frequently had the ``mode switch problem'': they did not know which mode they were in and made mistakes as a result. This was one of the reasons we reduced the number of modes dramatically.

\textbf{Palette.} The participants found the palette helpful for loading pre-existing graphics. The participants imported all fifty images of the people they interviewed into the palette, and used the images from the palette across several canvases. However, they found it difficult to find items in the palette when they loaded all fifty images into it, as the palette displays saved scraps as small snapshots, making them difficult to discern.

\textbf{Grid / Intentional interfaces.} During the majority of their usage, the interaction design group did not have intentional interfaces, but instead had the grid. They reported that they often moved to the grid perspective in order to get a high level perspective of their work. One of the designers reported that ``it's important for me to see everything at once''. The interaction design group used intentional interfaces for a brief period, but had difficulty adjusting to using it as a method of moving between canvases. They had trouble using the breadcrumb bar to navigate between canvases, and found the preview of the canvases within the cluster view too small. This, in a way, is not surprising given the brief exposure.

\textbf{Fading highlighter.} The interaction design group reported positive feedback about this feature, however, they seldom used it in practice. They reported the feature would be highly useful in working with larger groups, but all of their sessions took place with at most two people. Further, they found the name confusing as it did not function as they expected a regular highlighter would, in which a marker changes the color behind written text. Instead, they colloquially referred to it as the ``John Madden mode'', in reference to the sports announcer.

\subsection{Canvas navigation}

\textbf{Using perspectives to build personas.} The interaction design group used multiple canvases in order to organize their interviews by different perspectives. The people that they interviewed did not necessarily interact with the target software system the same way, but rather had different roles in interacting with the system, such as the customer or vendor, where each experienced different parts of the system. The interaction design group used various canvases for exploring the different groups, their dynamics with one another, and the emergent categories within each group. They performed this exploration by creating multiple copies of a template canvas that contained images of all fifty of the people they interviewed onto other canvases in order to explore different categorizations. For example, they organized the people they interviewed by location, finances, knowledge of technology, and similar stories. 

\textbf{Backing out to review work.} The interaction design group underwent cycles of intensive activity within a particular canvas, and bursts of consecutive movements between several canvases. In their initial exploration, they organized their image scraps along various one- or two-dimensional plots. After working in these canvases, the usage logs showed that the interaction design group moved between a zoomed out view and the different perspectives across canvases in rapid succession, which they reported as reviewing their designs in progress.

\begin{figure}%
  \centering
  \subfigure[One-dimensional plot with scattered notes] {
      \label{fig:ixdgroup:session1:a}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ixd-group/rep1}
   } 
  \subfigure[People interviewed organized using a two-dimensional plot] {
      \label{fig:ixdgroup:session1:e}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ixd-group/rep2}
   }
  \subfigure[Three-dimensional organization juxtaposed against two-dimensional organization] {
      \label{fig:ixdgroup:session1:f}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ixd-group/rep3}
   }
  \subfigure[People interviewed organized by table, Euler diagrams, and colored tags] {
      \label{fig:ixdgroup:session1:b}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ixd-group/rep4}
   }   
  \subfigure[People interviewed organized into spatial clusters with labeled topics] {
      \label{fig:ixdgroup:session1:c}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ixd-group/rep5}
   }
  \subfigure[Story-driven flow chart of people's experiences] {
      \label{fig:ixdgroup:session1:d}     
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/ixd-group/rep6}
   }   
   \caption {Various visual representations used by the interaction design group to create personas. \textit{Note: content in sketches does not depict actual work by the interaction design group, but instead are fictional examples of types of representations that they used.}}
   \label{fig:ixdgroup:session1}
\end{figure}%

\subsection{Representations}

The interaction design group created several representations in order to help them categorize their data. 

\textbf{One-dimensional plot.} When the designers began their design session, they created a one-dimensional plot, as in Figure \ref{fig:ixdgroup:session1:a}, to categorize the people they interviewed by a particular quality. They began writing names of individuals along the plot, grouping those individuals into lists, and marked the plot's horizontal axis with qualities at intervals between the lists of names. Additional text was written at each interval, or at edges, describing additional information about the qualities.

\textbf{Multi-dimensional plot.} In Figures \ref{fig:ixdgroup:session1:e} and \ref{fig:ixdgroup:session1:f}, the interaction design group used a two-dimensional plot to further organize their image scraps. The interaction design group first began with a one-dimensional plot that depicted a range of behaviors within a particular quality. They then placed the images and names of the individuals they interviewed along this plot. Afterward, they converted the sketch into a two-dimensional plot by drawing a line down the middle, and then further grouping the images along the new plot. Within the canvas depicted in Figure \ref{fig:ixdgroup:session1:f}, they juxtaposed a three-dimensional triangle plot next to a two-dimensional plot, both of which categorized a different set of scraps.

\textbf{Organization by plots, tags, and Euler notation.} After copying scraps onto the canvas, they used several techniques to simultaneously overlay  different layers of categorization. Within Figure \ref{fig:ixdgroup:session1:b}, the interaction design group used three types of categorization. First, they used a table. When the designers first entered this canvas, they initially dropped image scraps of those that they interviewed onto the canvas to view, and immediately afterwards drew a one-dimensional line and partitioned it into segments, similar to the canvas in Figure \ref{fig:ixdgroup:session1:a}. This time, however, they extended the one-dimensional line into a table by drawing long vertical lines and categorized the scraps within these spaces. Second, they used Euler diagram notation to depict emergent groupings by circling a set of image scraps and writing the name of the category next to the grouping. For example, the two image scraps in the top left have a yellow circle around them with the text ``NO CONDITION''. Third, they used color tags to denote another level of grouping. On the far left side, they created a legend with mappings between color and a category name. They then tagged the image scraps with color. The result was a set of image scraps that were organized using a table, Euler diagrams, and color tags.

\textbf{Spatially clustering objects.} In Figure \ref{fig:ixdgroup:session1:c}, the designers allowed the categories to emerge from the individuals they interviewed in a bottom-up fashion. In this canvas, they first wrote the topic of the canvas, which was ``Design Behaviors'', and grouped the faces of the people interviewed into categories based on their experiences, and then wrote the name of the grouping, such as ``ENJOYS TO EXPLORE'' or ``SEEKING OTHER TOOLS''. The interaction design group first wrote the topics using the regular pen, then later converted the topics into text scraps. The interaction design group further copied and pasted their notes from that session, and annotated with a dash to indicate that it was an attribute of a cluster. The interaction design group similarly spatially clustered their image scraps in Figure \ref{fig:ixdgroup:session1:d}, but additionally used arrows, and in one case formal connectors between scraps, to associate an image scrap with more than one group.

\textbf{Story-driven flowchart. } In one particular instance, the interaction design group walked through the customer's experience by drawing a story, as in Figure \ref{fig:ixdgroup:session1:d}. From the stories they gathered in their interview, they drew the different events that occurred during an average customer's purchasing experience, and noted possible options that may occur at each point. In order to construct the story, they reused elements they had stored in the palette, such as the image of a stick figure to indicate actors, and various icons to represent events, such as a phone to call a customer after a pending order was fulfilled. For particularly interesting events and corner cases in the story, the interaction design group placed the image of the person they interviewed to add to the story. 

\textbf{Storyboards.} The interaction design group created storyboards that demonstrated user interactions in a step-wise fashion. The interaction design group drew boxes that contained a picture of each step in a chain of actions, a set of descriptions for that image, and call-outs explaining pieces of the drawn picture.

\subsection{Collaborative work}

\textbf{Sharing control.} The interaction design group reported that a motivator for using Calico was to share control of content on the whiteboard. They further reported that they typically fall into the roles of either producing content at the whiteboard while the other participates from their laptop and provides design critiques of the content produced. When they did use Calico, the interaction design group, for the most part, continued their traditional roles, where the non-sketching interaction designer provided critiques while developing personas. However, while developing storyboards, the interaction designer taking on the non-sketching role was pro-active in typing out notes for each storyboard frame, entering both text and copy-pasting his pre-existing notes from an excel spreadsheet.

\section{Results: Deployment at a distributed software-based research team}
\label{chapter:evaluation:deployment3}

The research group used Calico over a period of five months in their meetings and design sessions. The high-level goal of their research was to develop an online solution for crowdsourcing the writing of software programs. Prior to using Calico, the team had already developed a software prototype, and used a whiteboard to discuss their designs, which consisted of process flows. They reported that their design had become too complex to perform solely on the whiteboard, and believed that Calico could help them evolve the design while still maintaining the informality of the whiteboard. The research group turned to Calico to ``work out the details of the process flow diagram once and for all''. At the beginning of the five month period, the remote member of the team flew in locally for a five day intensive session in which they created a grand design of the entirety of the system. They continued their collaboration remotely the remaining time. 

\subsubsection{Overview of design sessions}

The research group created several dozen canvases in their months of usage, using it to record meeting notes, design the underlying software architecture, and sketch out the user interface. The research group reported creating two major design diagrams, which consisted of a process-flow chart of the entire system, depicted in Figure \ref{fig:researchgroup:a}, and a second flow-chart for testing user-generated code in Figure \ref{fig:researchgroup:b}. Both were created during the initial five day session, and were continuously updated afterward. In creating the process flow in Figure \ref{fig:researchgroup:a}, the research group first sketched out the system as it existed at the date of the meeting. They then continued designing the parts of the system that did not exist yet. After four months of usage, they chose to perform a major refactoring of their system and the generated diagrams were no longer current. However, the research group reported that they continued to reference the old figures.


%Original design predated calico. Originally had it as a sketch in his office. ``What kind of process flow diagram was this?''. Ben towne came to town... they had the design machine, but it wasn't really worked out or designed in any sense. The motivation was that they wanted to sit down and work out the details of the process flow diagram once and for all. The design was their effort to sit through all the different sort of states that could be there. All the transitions. They spent a few days doing edits on their diagrams. Canvas1 was a process flow diagram for functions. They have another state machien for tests. The code already existed... so all through the week they would look back at the diagram, and figure out what was implemented and what wasn't.
%
%From the interview, the researchers created two major designs using Calico. The first was a process-flow chart taken 
%
%They wanted a way to add a layer so they could easily annotate the diagrams without messing up what was there. They instead kept track of everything in their heads...
%
%The diagram was already becoming very busy, it became hard to change it. It became an archival view

While two members of the research group were concerned with designing the system in its entirety, a third joined the research project with the intention of focusing on one specific aspect of the project, specifically how debugging is performed by the end-users of the system. Having joined the project after a prototype was already created, this additional group member needed to be onboarded \ref{cherubini2007let}. Throughout this process, he worked with the other members to become acquainted with the project by using Calico to sketch components of the infrastructure relevant to his tasks, and further build his project on top of the existing pieces of the system. In his role, he created sketches pertaining to his part of the system (Figure \ref{fig:researchgroup:c}), and additional sketches to help him understand the system (Figure \ref{fig:researchgroup:d}).

%- Another major member of the project joined four months into the project.
%- Extended system into his aspect.
%- used calico for onboarding.

\begin{figure}%
  \centering
  \subfigure[Process flow that described the main functionality of the software system] {
      \label{fig:researchgroup:a}
      \includegraphics[width=14cm,keepaspectratio]{./figures/Evaluation/researchgroup/canvas1}
   }  
  \subfigure[Process flow representing flow of how user submitted programs are tested] {
      \label{fig:researchgroup:b}
      \includegraphics[width=14cm,keepaspectratio]{./figures/Evaluation/researchgroup/canvas2}
   }
   \caption {Representations created by the research group during their one-week intense session.}
   \label{fig:researchgroup:1}   
\end{figure}%

\begin{figure}%
  \centering
  \subfigure[Process flow representing content from Figure \ref{fig:researchgroup:b} juxtaposed against a table representing another view on the process flow] {
      \label{fig:researchgroup:c}
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/researchgroup/canvas3}
   }
  \subfigure[Screenshot of live system used to design future versions] {
      \label{fig:researchgroup:d}
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/researchgroup/canvas8}
   }
  \subfigure[Members of the research group copied code into Calico to understand and revise system] {
      \label{fig:researchgroup:e}
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/researchgroup/canvas6}
   }
  \subfigure[User interface mockup of next version] {
      \label{fig:researchgroup:f}
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/researchgroup/canvas7}
   }      
   \caption {Representations created by the onboarded researcher that were used to better understand the existing software system.}
   \label{fig:researchgroup:2}   
\end{figure}%

\subsection{Feature usage}

\subsubsection{General feedback}

Overall, the research group found Calico to be a tool that supported them in carrying out their meetings. Prior to using Calico, the group used whiteboards to conduct their meetings, and would email pictures taken of the whiteboard among the group for those that were remotely located. They found that Calico allowed them to create more complex diagrams than they would have on the whiteboard. They further reported that for much of their design process they preferred to create the diagrams in Calico rather than a formal tool because they wished to maintain the informal feel of the whiteboard, and the ability to draw around the diagrams as needed. They did, however, recreate their major diagrams in Calico in more formal tools in order to document their designs.

All members reported that they felt more comfortable sketching lots of diagrams because of virtually unlimited space. All group members valued having their old designs readily accessible, and with some framing of their context due to intentional interfaces. One group member reported that he wanted ``to keep a history of what [I]'ve done, the branches that [I]'ve pruned. If you're designing complex things with stages, [you] need to tell a story''.

One of the members reported that it served as a lightweight thinking tool for him. The third reported that he typically used lightweight note-taking tools such as Evernote, which helped his individual thinking process, and later shared his work with the group. He reported that Calico offered more flexibility in what he could create, and found the flexibility helpful in preparing content before sharing it in group meetings.

%`we've already tried that, would you like to see where we've gone? It's already here''.

They reported some difficulties with Calico. Group members reported that they sometimes had difficulty launching Calico, or setting it up on a new machine was sometimes an obstacle with collaborators. Also, while sketching was adequate, the drop in writing quality was sometimes distracting. They often overcame this by using text scraps, but this was not always the case. 
%They stated that Calico lacked necessary formal objects, which they compensated for by created detailed sketch objects and copying them.

%- sort of like evernote. He uses it like he would evernote, because he's most of the time alone, then shares it with the other group memebers
%- ``designs get very complex''. 

%In interviews, I asked users if using Calico and its features were invasive to their design activity...

%When asked if they made any compromises in order to perform their activities using Calico, 

%their goal was to just draw something up. They thought about it in terms of a freehand drawing space.

%- 


%- sketching is a bit off
%- moving things is one of the largest positives


%- one member likes to work in structures

\subsubsection{Feature specific}

%When asked about the features as a whole...

\textbf{Basic sketching and features.} The research group reported that the basic sketching features were adequate for their design sessions, although they reported that the electronic whiteboard made their handwriting messy. They further reported that some of the menu buttons were not clear in their functionality. They expected the ``navigate backwards'' and ``navigate forwards'' buttons to move back and forth in links within the intentional interface rather than navigating the history of visited canvases. They further thought the ``clear canvas'' button was the eraser mode button, causing them to accidentally clear their canvas, which they recovered by pressing ``undo''.

\textbf{Scraps.} The research group made frequent use of scraps across their design sessions. They found scraps and connectors useful because both made otherwise static elements interactive, such as when working with process flows. They reported feeling less afraid to create complex diagrams because the elements could be moved.  They could create space by moving scraps, and also categorizing objects, such as when placing them in tables, like in Figure \ref{fig:researchgroup:c}. 

They encountered some difficulties using scraps. Scraps were difficult to select when many were overlapping. Scraps could not have their text changed, which required the research group to delete scraps and their connectors when text needed to be changed. They further desired the press-and-hold selection gesture to trigger more quickly.
%Encountered problem when they had multiple scraps in one area and tried to select it. They had layered annotations, and that made selecting things a problem.
%- issues where they couldn't edit the text
%- adding connectors was a pain, because editing required deleting, and recreating it.
%
%- wrestled with trying to draw sometimes, selecting
%
%Connectors very important. Referencing arrows very important. Nice that arrows could lock on, and arrows would move while scraps would drag.
%
%Creating and moving scraps sometimes left you with a line. It works some of the time, but it's not predictable whether it's going to work or not going to work.
%
%Adding text is not intuitive, stumbled upon the keyboard shortcuts by accident.
%
%- preferred scrap interaction over lasso 
%
%ngs within a canvas
%
%- overall he enjoyed his experience
%- he wanted a way for press-and-hold to be faster

\textbf{Palette.} The research group did not use the palette in their sessions. Rather, when using repeated elements, they copied canvases and deleted non-relevant content. They also did not use repeated elements in their sketches, but rather always sketched new content, or reused old sketches.

\textbf{Intentional interfaces.} The research group made heavy use of the breadcrumb bar to move between canvases. They found that using the breadcrumbs was the easiest way to move between canvases. They reported that canvases were too small to visually identify in the cluster view, where canvases in a circle with thirty or more canvases became very small. Identifying clusters by name in the breadcrumb bar was easier in comparison.

Two members of the research group also did not use tagging with intentional interfaces because they found the system of tagging to be confusing. They found grouping canvases by clusters helpful because it allowed them to separate content between members. However, within clusters, they did not link canvases. The third researcher, in contrast, used tagging to link canvases into linear chains, each of which represented a topic. He reported that many canvases by themselves were difficult to return back to and read without context, but if viewed one after the other, one could more readily understand the concepts in juxtaposition to the previous canvas. On the other hand, he found the act of tagging canvases as ``alternatives'' or ``abstractions'' awkward because he was not sure what those meant in the context of his sketches.

%- he did not use tagging because he didn't really understand it.
%- useful if he could tag pieces of this
%- A problem is the performance
%- difficult to read things out of context. But if they look at canvases one after the other, it makes it easier to understand concepts that extend the previous ones.

\textbf{Fading highlighter.} The members made light use of the fading highlighter. They reported that, while it was useful on occasion, they preferred to talk out loud and make pointing gestures using their hands. In remote sessions, the group member from CMU relied on video communication to see these non-verbal cues. Though they found the fading highlighter helpful, they oftentimes forgot that it was available. They reported that they would have used it more if it had been easier to invoke.

\subsection{Canvas navigation}

During sessions involving the design of the process flow in Figure \ref{fig:researchgroup:a}, the research group reported that they seldomly moved to another canvas. Rather, they used additional tools such as Google Docs to record requirements and design decisions, or simply talked out loud. All sessions using Calico also involved the use of Skype and Google Docs.

\textbf{Using perspectives to support onboarding.} The research group member who underwent onboarding used canvases of several perspectives in becoming acquainted with the software system. This member used a mix of user interface mockups, freehand sketches, tables, and other diagrams to step through the implementation of the software already written and the design of their own new features. This initial exploration involved importing an image of the interface into Calico,  and stepping through the interface to inspect items pertaining to debugging user generated code. From these sketches, he switched to other canvases in which he generated the process flow in Figure \ref{fig:researchgroup:b}, which was another perspective of Figure \ref{fig:researchgroup:a}, but distilled to only include details relevant to his project. Later, while again exploring the process flow in Figure \ref{fig:researchgroup:c}, the group member created several copies of the canvas, preserving the process flow at the top of the canvas, and creating different tables, lists, and mock-ups below while experimenting with different algorithms.

\textbf{Exploring alternative process flows.} The same researcher that underwent onboarding drew several alternatives for his part of the system. He created several copies of the canvas in Figure \ref{fig:researchgroup:c}, which consisted of a process flow diagram juxtaposed against a table. Between these alternatives, the process flow depicted at the top of the canvas did not change. Instead, the table below served as a template to experiment with alternative paths for components. The group member frequently referenced his previous canvases to compare against other alternatives.

%Having recently used diagrams would have been helpful. Mostly used 1 or 2 diagrams, at most 4 to 5.

%- began with mockups. in every statement that used a callee function, they logged before, and after.

%- simplification of large diagram. From the large sketch, created a separate sketch that was a distillation of his own parts

\subsection{Representations}

The research group primarily used Calico to work on process flows, however they additionally used other types of representations.

\textbf{Lists.}  Not depicted, lists were used on occasion to record bullet points from meetings or during brainstorms. The research group used a mix of both written lists, which were used as scratch notes, and text scraps, which were rapidly created during brainstorm sessions. On the whole, lists were not used often because the research group took notes in Google Docs, and used Calico to supplement written documents.

Additionally, in some complex diagrams, the research group reported keeping track of ideas using lists. For example, in a sketch similar to Figure \ref{fig:researchgroup:c}, a group member reported writing down questions he had about states of the diagram, which he would investigate later. Similarly, the top of Figure \ref{fig:researchgroup:d} contains additional notes and questions while brainstorming a user interface.

%- came up with some questions while creating their design, noted them on the side.

\textbf{Timeline.} Not depicted here, timelines were used to record target dates for important dates, such as conference deadlines.

\textbf{Process flow.} The research group used process flows extensively, as the design of how tasks flowed from one person to another are critical to their design. As such, a lot of detail was put into their process flows in order to capture the details of the system.

The research group used process flows as their representation of choice to design their system. During their intensive five day meeting, they attempted to model the entirety of their software onto one canvas, calling it a ``a view onto their system''. As the design grew, they encountered issues with available space within the canvas, but did not wish to break up their design across multiple canvases. The research group reported that they ``thrashed about how much detail they should put in the diagram''. As their design grew, they shifted from using plain sketches with drawn arrows, to using scraps with connectors. They found text scraps easier to read than their own handwriting and more space efficient. Also, they found the ability to have connectors remain attached to scraps as they were moved very useful, as it allowed them to make space for more states, or place similar states in close proximity to one another. After the five day session, they reported a resistance to changing the diagram because of its complexity. They ``knew where everything was because it had always been there'', and when they did make additions, those parts of the diagram ``grew organically'' by occupying vacant white space without displacing the positions of the scraps around it.
% - it's a view onto their system, and they think about the correspondence between all of their views

They encoded additional information into their diagrams by using color, as depicted in Figure \ref{fig:researchgroup:a} . One research group member stated that they ``needed some way to encode what was going on'', where some scraps represented states, some represented tests, and some as points of branching. For example, black connectors represented a state transition with no information passed, blue represented the passage of tokens, and so on. The research group reported some difficulty in recalling the meaning of all colors for connectors. They further distinguished the scraps representing states that were not yet implemented by tagging them with color. In the upper right of Figure \ref{fig:researchgroup:a}, they wrote ``Red: not yet implemented'' in red, and tagged scraps with this quality by underlining them with red. 

%Tagging.
%- annotated stuff with categories (what was implemented, what was not)
%- blue represents sending a whole artifacts.
%- used pen color to declare different types of connector
%- needed some way to encode what was going on. Some boxes were states, some are tests, some are particular branches

When the research group described the diagram, they saw it not as a complex network of individual states, but rather as a set of higher-level structures that represented different parts of a complex work flow. That is, they implicitly grouped parts of the diagram into larger components. Their discussions took place over different regions of the diagram, which they understood among themselves to represent components and verbally referred to them by name, but did not record within the process flow because they were not sure how to represent them. When talking about features they requested a way to declare ``these are the typical paths'' or ``these two paths are similar''. When their discussions took place, they discussed specific workflows within the diagram, and discussed what happens in those workflows. The research group struggled with how to abstract away these workflows, and created additional diagrams, such as diagram depicted in Figure \ref{fig:researchgroup:b}.

%Paths
%- 
%- ``layering'' was an important concept.  They wanted to declare ``these are the typical paths'' or ``these two paths are similar'',
%-  ``annotate that particular paths have a particular meaning''.
%- were trying to encode different workflows, what happens in different situations
%- became difficult to keep track of all the paths
%- thought about ways to abstract what was going on

%Figure \ref{fig:researchgroup:1}
%- didn't know how to use connector feature

%- simplification of large diagram. From the large sketch, created a separate sketch that was a distillation of his own parts

\textbf{Table.} One of the research group members used tables to help him process sketches while onboarding. He created tables such as the one depicted in Figure \ref{fig:researchgroup:c}, and juxtaposed them against simplified sketches of process flows in order in support of a detailed examination. The process flow in Figure \ref{fig:researchgroup:c} contained several paths that could be taken, and the group member created tables to capture a particular workflow. He did so by listing each state in the row header, transitions along the column headers, and placing scraps in the intersecting cells. He created several such tables to capture different workflows.

One of the group members reported that they created the table in Figure \ref{fig:researchgroup:c} to examine a set of corner cases. The table served as an aid to help them think through the process flow. The group member reported that they used the table to simulate an algorithm, thinking through ``how this would be executed''. They reported that scraps were helpful in being able to move things. They reported that they normally would have created this diagram on a whiteboard, but in comparison, Calico provided ``lots of space'', and made their table ``very clean'' because they did not have to redraw things. 

%``Dynamic execution of [his] diagram''. Would have enjoyed being able to play states.

One of the group members, in contrast to the others, used Calico more frequently than the others to help him design while working alone. He would prepare several canvases of sketches prior to a meeting, sketches including lists, tablets, process flows, etc., to explain his designs. He expressed that he preferred to make his designs within Calico both because the other members were already using Calico, and because sharing sketches in Calico was faster because it was already a shared space.

%- Examined corner cases. Came up with different solution. Created mockups. Looked at dependencies at subcalls.
%
%- worked alone, and used canvas to help himself think through. Used table to help him think through the points of failure in the process flow.
%
%- Looked for corner cases
%
%- u

\textbf{User interface mockups. } The research group created prototypes of the user interface late in the design cycle, after having already created a working system that they could interact with. They used mockups during discussions to suggest changes to the user interface, and when designing new web pages. During these sessions, they imported screenshots of their current interface into Calico as a visual reference, and sketched interfaces without the use of scraps.

The onboarded group member created user interface mockups within Calico in order to help him in thinking through his design. In his task, he used a combination of screenshots and sketched mock-ups to first walk through how existing end-users perform their task and submit their finished work. From these sketches, he created a process flow to capture the interaction of the user and the flow of information.

%- began with mockups. in every statement that used a callee function, they logged before, and after.
%
%- difficult to think through the problem
%- created mockups of what they wanted the user to create
%- ``similar to building blocks''.
%- they wanted to create a visual frontend for building a user interface
%- problem with people programming things as event listeners... harder to build his tool around that

%- broken up into different structures. Some with arrows.

\textbf{Spatially clustered text scraps. } In two meetings, the research group generated a set of text scraps as part of a brainstorm, and later grouped them into similar topics. They did not use formal arrows, but rather used grouping symbols, such as Euler diagrams, and drew arrows between the groups to indicate different types of relationships. 

\textbf{Dendrogram.} One researcher created a horizontal dendrogram to explore a set of questions generated from other sketches. The researcher began with a set of four questions, and from those questions, expanded into a tree of questions to explore a problem. They drew arrows to link items between the questions, as well as several freehand annotations.

\textbf{Source code.} In several of the meetings, one of the group members pasted screenshots of source code into Calico to discuss his implementation. He pasted both psuedo code and actual Javascript code. In meetings, he drew call outs from the screenshots to explain each method call, as in Figure \ref{fig:researchgroup:e}. 

\subsection{Collaborative work}

The research group conducted meetings using the two electronic whiteboards as well as their own laptops. During co-located meetings, they sometimes loaded both boards with Calico, but visually viewed different canvases on each whiteboard, or used the second board to display Google Docs. They reported that only one person usually sketched, while another took notes on in Google Docs, and any remaining participants simply talked. 

%During meetings, one person mostly sketched.
%- They used both boards. Second board had either second diagram (canvas 2), google doc, or skype face. Almost always the case where they edited just one of the canvases.
%- They mostly talked through particular scenarios. One person was in charge of drawing everything.

\textbf{Helping remote members ``feeling connected''.} In distributed meetings, remote members reported feeling ``more connected'' when using both Calico and video chat software. In these meetings, members often used another display loaded with either Skype or Google Hangout. The remote participant reported that the ideal setup included a camera that pointed at the boards with Calico, because it allowed him to observe the body language of the speakers. They reported that this setup was particularly helpful during the early phases of their collaboration because people often pointed to objects on the boards, gestured to content at the board from their seat, or gestured in free space while explaining an idea. The combination of both being able to see the body language of remote members and being able to manipulate that same content in real-time culminated into the sense of ``connectedness''.

%Remote member felt connected.
%- When person left, they could reference things pretty easily.
%- particularly useful use case was when the camera was in the back, and pointed at both the calico board and the skype board, and it showed where people were pointing at. They had body langauge, what they were pointing at, etc. 
%- Body langauge is important in the early phases, because people are pointing, gesturing, using hands in their explanation of meaning.

\textbf{Continuity between design sessions.} The research group also reported that Calico made it easier to stop and resume design sessions. One member reported that the ``biggest benefit was crossing space and time'', in which Calico provided a consistent virtual space to conduct meetings prior to the five day visit, during, and long after. All canvases remain in a consistent location that all group members can always access. They did not need to worry about cleaning up any materials in their space or sharing whiteboard space with other research groups conducting meetings in the same space. 

%Starting and stopping.
%- ``Biggest benefit was crossing space and time''. Being able to ``put things on pause and resume''. Content was mobile across rooms, and able to stop and start activities again next time because content was where they left it.

\textbf{Providing an archival reference.} After the diagrams became out of date, the research group reported that they used Calico as an ``archival reference''. The research group originally thought that they would continuously interact with their sketches, but instead used their sketches as visual references. The team eventually referenced the sketches less and less as they had internalized the contents of the sketches and could reference freely without the need to have the sketches in front of them. However, they reported that they valued having the sketches in Calico rather than static photographs of whiteboards. They reported that it gave the sketches ``a sense of permanence'' because they could return and make adjustments to the sketches at any time, which they did. After undergoing a major refactoring, the content of the sketches in Calico became outdated, however, they did reference them still at times while moving forward. They reported that it was ``good to have a reference of what the architecture was like in the previous version''. They went back to past designs to see what they did before. When looking back at previous designs, they noted that ``it would have been nice to have annotations as to why this [previous design] wasn't going to work''. They remarked that while their sketched designs were most useful during implementation, they ``would turn back to the diagram because they forgot how they implemented something while writing''.

%Used Calico as an archival reference.
%- The team internalized their sketches from Calico, and pulled up sketches occasionally as an archival reference.
%- Thought it would be more interactive, but it wound up being visual reference.
%- It was more about having this shared representation that was visible, and stuck around longer.
%- Did have advantage over picture because they could tweak it.
%- It wasn't finalized until it was in the paper
%- It was more helpful during implementation, but would turn back to the diagram because they forgot how they implemented something while writing.
%- refactoring did not get reflect in Calico. However, they did reference it again while moving forward. It was good have a reference of what the architecture was like in the previous version. They went back to past designs. 
%- ``It would have been nice to have annotations as to why this wasn't going to work''.
%- It would be more useful to be able to add notes within the context of sketches, to declare rationale of a specific spot.

%- they created one whole design, that was later thrown away. The design persisted, but they said that it was thrown out. Interesting that the designs still remained

\section{Design behaviors}
\label{chapter:evaluation:design-behaviors}
In this section, I turn to examining the design behaviors across all three sessions. I report on whether they occurred or not, and whether the features of Calico were used as intended. This is not an exhaustive review, i.e., I am not reporting on every instance of each design behavior occurring. Rather, I am looking for proof of existence. This breaks down into three parts: (1) did I see proof of the designers performing the design behaviors, (2) did they use the features of Calico to perform them, and (3) if so, which features.

%Start this section with a table. Summary of findings. Talk about each row of the table, because each row is a design behavior, in a little more detail
%Each sub section: design behavior... explanation. Now, rather than defending each one. 

\subsection{Design Behaviors Summary}

Based on evidence from usage logs, as corroborated by the interviews, all design behaviors were observed as occurring at least once. More specifically, all 14 design behaviors occurred in the OSS group, 9 of the design behaviors occurred in interaction design group, and 13 of the design behaviors occurred in the research group. With respect to the usage of features, 13 of the 14 design behaviors for the OSS group were performed using Calico's features, 5 of the 9 design behaviors for the interaction design group were performed using Calico's features, and 12 of the 13 design behaviors were performed using Calico's features for the research group.

Table \ref{chapter:evaluation:designbehaviors-table} summarizes the features that supported each design behavior, which included scraps \& connectors, palette, intentional interfaces, and fading highlighter. Referring back to the table of features in Chapter \ref{chapter:calico-version-two} (Table \ref{table:calico-version-two:designbehaviors}), each feature was designed to support a specific set of design behaviors. In Table \ref{chapter:evaluation:designbehaviors-table}, all non-targeted features are shaded in gray; that is to highlight when features were used in support of behaviors they were not explicitly designed for.

A summary of the design behaviors as they occurred is presented as follows:

\begin{enumerate}
	\item \textbf{They drew different kinds of diagrams.} All groups performed this behavior using scraps, as well as with plain sketching as on regular whiteboards without using Calico features.
	\item \textbf{They produce sketches that draw what they need and no more.} All groups created scraps that were either sparse in details or notationally rich, depending on their needs. They also did so with plain sketching when not using scraps.
	\item \textbf{They refine and evolve their sketches over time.} The OSS group refined sketches into box-and-arrow diagrams using scraps. The interaction design group refined one-dimensional plots into both two-dimensional plots and tables using regular sketching.
	\item \textbf{They use impromptu notations.} Both the research group and the interaction design group created custom notations using scraps and connectors. The interaction design group created custom icons using the palette. The OSS group created their own visual language using plain sketching by creating connectors with custom colors, shapes, and textures.
	\item \textbf{They move from one perspective to another}. All three groups used intentional interfaces to move between canvases of different perspectives. The OSS group and the research group benefited from intentional interfaces because chaining canvases provided an ordering for the canvases, which helped members of each group remember the meaning of their sketches within canvases. All groups also moved between perspectives within a single canvas.
	\item \textbf{They move from one alternative to another.} The OSS group and the research group used intentional interfaces to copy canvases and label them as alternatives. All three groups discussed alternatives verbally without using Calico.
	\item \textbf{They move from one level of abstraction to another.} The OSS group and the research group used intentional interfaces to step between canvases of different levels of abstraction. The OSS group used scraps to represent the software components they ``stepped into'' and ``out of''.
	\item \textbf{They perform mental simulations.} The OSS group and the research group used the fading highlighter to walk through the flow of data in their designs. The interaction design group created elaborate sketches that walked through the story of a user using regular sketching, but did so without any of the advanced features of Calico.
	\item \textbf{They juxtapose sketches.} The OSS group and the research group used scraps to position sketches next to one another for reference. The research group used the palette to copy content across canvases so that they could reference it. All groups also drew sketches of different perspectives in the same area to juxtapose them.
	\item \textbf{They review their progress.} All groups reported that they used intentional interfaces to review the sketches they have created (and the grid in case of the interaction design group). The OSS group and the research group also created handwritten lists that summarized the designs that they recently created.
	\item \textbf{They retreat to previous ideas.} The OSS group and research group both used intentional interfaces to return to past designs and design decisions. The research group also returned to designs that were several months old.
	\item \textbf{They switch between synchronous and asynchronous work.} The OSS group and the research group had members that used intentional interfaces to deviate from the group discussion to their own canvas to reference content or create alternatives. However, though it did occur, it occurred relatively rarely in the research group. 
	\item \textbf{They explain sketches to each other.} The OSS group and research group used the fading highlighter to explain designs. The OSS group reported finding this very useful to discuss data that passed between software components. 
	\item \textbf{They bring their work together.} The OSS group created new canvases that combined work from previous canvases.
\end{enumerate}

\begin{center}
\begin{longtable}{|p{5cm}|p{5cm}|c|c|c|}
\caption{The set of design behaviors and the features that supported them}\\
\hline
\textbf{Design Behavior} & \textbf{Supporting Feature} & \textbf{OSS} & \textbf{IxD} & \textbf{Res} \\
\hline
\endfirsthead
\multicolumn{4}{c}%
{\tablename\ \thetable\ -- \textit{Continued from previous page}} \\
\hline
\textbf{Design Behavior} & \textbf{Supporting Feature} & \textbf{OSS} & \textbf{IxD} & \textbf{Res} \\
\hline
\endhead
\hline \multicolumn{4}{r}{\textit{Continued on next page}} \\
\endfoot
\hline
\endlastfoot
\hline
1. They draw different kinds of diagrams&Scraps \& connectors &X &X &X  \\* 
\hhline{|-|-|-|-|-|}
%\hhline{|~|-|-|-|-|}
2. They produce sketches that draw what they need, and no more&Scraps \& connectors &X &X &X  \\\hhline{|-|-|-|-|-|}
3. They refine and evolve their sketches over time&Scraps \& connectors &X & &  \\\hhline{|-|-|-|-|-|}
\multirow{2}{5cm}{4. They use impromptu notations}&Scraps \& connectors & &X &X  \\\hhline{|~|-|-|-|-|}
&Palette & &X &  \\*\hhline{|-|-|-|-|-|}
\multirow{2}{5cm}{5. They move from one perspective to another}&Scraps \& connectors &X & &X  \\\hhline{|~|-|-|-|-|}
&Intentional interfaces &X &X &X  \\*\hhline{|-|-|-|-|-|}
\multirow{2}{5cm}{6. They move from one alternative to another}&Scraps \& connectors &X & & \\*
\hhline{|~|-|-|-|-|}
&Intentional interfaces &X & &X  \\*
\hhline{|-|-|-|-|-|}
\multirow{2}{5cm}{7. They move between levels of abstraction}& Scraps \& connectors &X & & \\*
\hhline{|~|-|-|-|-|}
&Intentional interfaces &X & &X  \\*
\hhline{|-|-|-|-|-|}
8. They perform mental simulations&Fading highlighter &X & &X \\\hhline{|-|-|-|-|-|}
\multirow{2}{5cm}{9. They juxtapose sketches}&Scraps \& connectors &X & &X  \\*
\hhline{|~|-|-|-|-|}
& \cellcolor[gray]{0.8}Palette & \cellcolor[gray]{0.8}& \cellcolor[gray]{0.8}& \cellcolor[gray]{0.8}X  \\*
\hhline{|-|-|-|-|-|}
10. They review their progress&Intentional interfaces &X &X &X  \\*
\hhline{|-|-|-|-|-|}
11. They retreat to previous ideas&Intentional interfaces &X & &X  \\*
\hhline{|-|-|-|-|-|}
12. They switch between synchronous and asynchronous work&Intentional interfaces &X & &X  \\*
\hhline{|-|-|-|-|-|}
13. They explain their sketches to each other&Fading highlighter &X & &X \\\hhline{|-|-|-|-|-|}
\multirow{2}{5cm}{14. They bring their work together}& \cellcolor[gray]{0.8}Scraps \& connectors & \cellcolor[gray]{0.8}& \cellcolor[gray]{0.8}& \cellcolor[gray]{0.8} X \\*\hhline{|~|-|-|-|-|}
&Intentional interfaces &X & & 
\label{chapter:evaluation:designbehaviors-table}
\end{longtable}
\end{center}

\subsection{Kinds of sketches software designers produce}

\subsubsection{Design Behavior 1: They draw different kinds of diagrams}

Across all three field deployments, the developers and designers created several representations using different types of notations, or approximations of notations. All groups used lists, the OSS group used box-and-arrow diagrams, the interaction design group made heavy use of one- and two-dimensional plots, and the research group used process flows and tables. Most of the canvases across all of the sites used only one type of notation, but a few of the canvases within each group also contained a mixed set of representations using different notations. This difference compared to the experiments in Chapter \ref{chapter:calico-version-one} and \ref{chapter:notation-paper} came about because, unlike the controlled experiments in which the entire design session occurred at the whiteboard with no other materials, the designers in-the-field turned to Calico for specific tasks, such as when the OSS group created sketches to plan the explanation of an architecture, which may lead to canvases with more targeted content. Longer term use of Calico may lead to using it for more in-depth tasks, designers may supplement it with other tools such as word documents, their own computers, and so on. Designers may also need to create fewer sketches since they may have internalized many of the representations they sketched in previous meetings, which the research group reported to be the case for them.

The OSS group and research group created many different types of representations using scraps. The OSS group used scraps to create box-and-arrow diagrams and user interface mockups while brainstorming the elements and look of a GUI, as depicted in Figure \ref{fig:ossgroup:session1:a}, and mixed lists with user interface mockups in \ref{fig:ossgroup:session1:b}. In each group, creating the representations in the same space allowed them to evolve both concurrently, where the box-and-arrow and list representations allowed the developers to consider the components from a structural perspective, and the mockups allowed them to consider the design from the end-user's perspective. They reported that, by depicting these elements as scraps, it made the elements easier to move and resize around the canvas, and therefore felt more like entities in their eyes. For the research group, one member of the research group mixed different types of representations in one of his canvases in order to help him think. In Figure \ref{fig:researchgroup:c}, the researcher used a table in order help him step through a process flow, and used scraps to move items around the table.

The interaction design group, outside of image scraps and text scraps, seldom used scraps to depict major concepts in their canvases, but did create different types of representations as they would have on a regular whiteboard. They used scraps to depict very basic concepts, such as image scraps to show the faces of those they interviewed and text scraps to produce legible text when their handwriting itself was not legible. However, when creating complex representations, they simply sketched as they would have on a regular whiteboard. The interaction design group placed graphs of different types next to one another, such as the one-dimensional and two-dimensional plots in Figures \ref{fig:ixdgroup:session1:a}, and the triangle graph and two-dimensional plot in \ref{fig:ixdgroup:session1:c}. 

%- Same canvas? Not often... Figure in OSS group. Figure in Interaction design group. Figure in research group (christian's!)

\subsubsection{Design Behavior 2: They produce sketches that draw what they need, and no more}

All groups engaged in this behavior, in which they rarely actually completed their sketches in all detail, and used full notational conventions sparingly. Evidence of this came from both the amount of formal conventions used in the sketches, and the disparity between the design as drawn in Calico and the designs as explained in the interviews.

First, all groups varied the amount of notational conventions used between sketches, even when those sketches were of the same representation type. The OSS group, for example, expressed most of their software architectures using only boxes-and-arrows, and labeled the connecting arrows in very few instances. In most cases, the OSS group discussed the design verbally and only added detail to their diagrams in order to have something to point at during discussion. The interaction design group used varying amounts of notational convention, sometimes labeling the axes in their plots in great detail, as in Figures \ref{fig:ixdgroup:session1:a} and \ref{fig:ixdgroup:session1:d}, other times including very little detail as in Figure \ref{fig:ixdgroup:session1:b}. The research group also engaged in this design behavior when working with their process flows, in which only some states contained entry and exit conditions, and many arrows lacked labels. 

Second, it became evident through the interviews that the sketches made in Calico omitted many important details of designs sessions. In interviews, members from the OSS group and the research group had difficulties identifying the meaning of some of the sketches they created, however they recalled the overall objective of the sketches, which they deemed more important than the individual details. These sketches were used to support activity while ``in the moment'', where sketches were used to accomplish an immediate goal such as brainstorming, explaining, or, in the case of the interaction design group, categorizing people interviewed. For example, a single developer in the OSS group created a sketch to explore use cases of a user interface (Figure \ref{fig:ossgroup:session1:e}), but only used the bare minimum detail to do so. The sketch itself is a distillation of previous sketches, Figures \ref{fig:ossgroup:session1:c} and \ref{fig:ossgroup:session1:d}, and does not use notations to suggest that it is a use case diagram. The source code in Figure \ref{fig:ossgroup:session3} is also a sketch used in an individual session with minimal detail. The developer invested the time to write pseudo-code, but used arrows with minimal explanation of their meaning. 

In the above cases, all groups used scraps to perform this design behavior, and the OSS group and interaction designer group used the palette as well. The OSS group reported that they used scraps to represent ``important entities'', which served as a low-detail representation that they could point to during design discussions and move around the canvas. They reused these low-detail entities across several canvases using the palette as well. The interaction design group used image scraps of people, which they saw as the lowest level of detail in their sketches. They additionally saved low-detail icons such as a telephone and computer to the palette, and placed them in their sketches. Their sketches did not explain these icons, but their meaning was known throughout the design discussion. The research group also used scraps to represent states in process diagrams, which they manipulated quite heavily. Across all groups, content that was not the subject of manipulation often did not become scraps.

The fading highlighter also played an important role in supporting low-detail diagrams for the OSS group. This group used the fading highlighter extensively to discuss components, draw flows of data, and discuss details of software components without permanently adding more details. By using the highlighter instead of actually drawing more detail, they explicitly chose to preserve sketches at a low level of detail.

% externalize an entity so that they can externally examine it, but sketching done in the examination includes a scarce amount of detail. 
%
%The sketches used to support individual thinking had few details and little use of formal notations.
%
%Of the sketches used to support personal thinking
%
%Sketches to support personal thinking. In gui, created something because it didn't look right. Didn't include much detail of rationale. In code, got important part on board, but didn't write beyond that, used arrows to help him think.
%
%Sketches to support group brainstorming. In group brainstorm, had more details.
%
%Sketches to support explanation to group. Explanation, had much fewer details.
%
%With respect to notational convention, we saw very little of that present.
%
%- minimal drawings in oss group. ``they only draw what they need'', very true for much of the OSS group. Diagrams support discussions, no need to declare details since they won't be executed! The things that get written down are the major entities. Objects for OSS group. pictures for interaction design group. state transition names for researchers
%
%- notational convention is almost non-existant. These are not deliveries, but rather, representations provide framing for conversations (thinking of OSS group).

\subsubsection{Design Behavior 3: They refine and evolve their sketches over time}

Two of the three groups exhibited this design behavior. In the case of the OSS group, they first sketched the names of entities, and these sketched names eventually became text scraps with connectors. The result of these sketches is depicted in Figure \ref{fig:ossgroup:session1:a}. In this sketch, the developers began with simple sketches, and increased their notational convention and amount of detail over time. In the case of the interaction design group, they began with pictures of faces, and across several canvases, categorized these pictures using visual structures such as tables and plots. Within Figure \ref{fig:ixdgroup:session1:b} in particular, the interaction design group evolved their representation by beginning with a single dimensional line, adding categories on that line, and transforming it into a table. The interaction design group did not set out to create the table, but rather created the table after categorizing the faces. Other representations also began as a one-dimensional plot, such as Figure \ref{fig:ixdgroup:session1:e}, which was later refined into a two-dimensional table.

The research group did not exhibit this behavior. This was because the research group began their major design sessions by copying existing diagrams from the whiteboard into Calico, and did not start design sessions from scratch, as was the case for the OSS group and the interaction design group in the sketches they refined.

The OSS group used the scraps feature to refine an existing sketch, but the interaction design group did not. Scraps helped the OSS group refine their sketches because scraps support the creation of box-and-arrow diagrams. The interaction design group refined their sketches into more complex plots, which scraps do not support.

\subsubsection{Design Behavior 4: They use impromptu notations}

All three groups performed this design behavior in which they created a visual language that was particular to their own respective designs. Each group made consistent use of their own devised notations, meaningfully using connectors, tagging, and self-drawn icons to layer meaning on structured diagrams.

The interaction design group and research group used scraps to create their own notations. Both groups used Euler diagrams to group scraps by drawing circles around sketches (used within Figures \ref{fig:ixdgroup:session1:b} and \ref{fig:researchgroup:b}, among other diagrams not shown). Each group tagged scraps using colors, where the interaction design group used color patches in Figure \ref{fig:ixdgroup:session1:b}, and the research group used colored underlines in Figure \ref{fig:researchgroup:a}. The research group, unlike the interaction design group, used color coded connectors with their scraps, which represented specific types of transitions between process flows. In the case of the colored connectors, the data types were informally known by the people who sketched them, but their meaning was not apparent from just the sketch alone. 

The OSS group also created their own notations, but did not use scraps to do so. Like the research group, the OSS group used colored arrows to represent different data types. However, the OSS group improvised representations in ways that regular Calico connectors could not, such as by using shape and texture to give meaning to their sketches. The arrows in Figure \ref{fig:ossgroup:session2:b} used a dashed texture to represent data, as opposed to solid arrows which were method calls between components. In Figure \ref{fig:ossgroup:session2:c}, the OSS group used colored boxes to represent events that were broadcasted to multiple components, but originated from the same source. Further, the member from the OSS group that created the psuedo code in Figure \ref{fig:ossgroup:session3} improvised several types of arrows that served as placeholders for concepts during his walkthrough. Interestingly, the member from the OSS group could not identify the meaning of his own improvisations afterwards, but reported that they helped his thinking during design.

The interaction design group further used the palette to create their own set of scrap icons. They created symbols such as a phone, a computer, and other symbols to represent actions, which they used used in their storyboards by placing the icons next to actors who could perform those actions.

%(examples include Figures \ref{fig:ossgroup:session1:a}, \ref{fig:ossgroup:session2:d}, \ref{fig:ixdgroup:session1:a}, \ref{fig:researchgroup:session2:b})

%- definitely, show the pictures from the source code thing. Show further notations from calico
%
%- yes, reappropriated, but only from plain sketches into boxes...

\subsection{How they use the sketches to navigate through a design problem}

\subsubsection{Design Behavior 5: They move from one perspective to another}

All groups shifted their focus among multiple canvases with different perspectives in their design sessions. Both the OSS group and the research group, in working with source code, moved between perspectives such as the user interface, software architecture, lists of requirements, source code, etc. The interaction design group, in contrast, used Calico to build personas from interviews, and moved between canvases that categorized the same data, but using different categorizations and visual structures (such as tables, one- and two-dimensional plots, etc.). 

The OSS group made heavy use of intentional interfaces in moving between perspectives. The OSS group chained canvases in their sessions, which provided them the benefit of ordering canvases such that a chain of canvases could convey a story. Sometimes this order reflected the chronology of their exploration in the design space, where the earliest canvas in a session was on the inner circular ring. However, canvases were sometimes inserted within the chain when they returned back to previous sketches and deviated to a new idea. Overall, having the canvases linked into groups helped them remember ``how the session played-out''. In some cases, members from the OSS group could not identify parts of their sketches, but were able to recall their meaning by examining the surrounding canvases in the chain.

The research group also used intentional interfaces, but to a lesser degree than the OSS group. In the research group's intensive five day session, they reported that they preferred to maintain a single diagram so that all components of their system were always visible in their meetings. They instead preferred to examine other perspectives using their own computer within Google Docs. One of the members of the research group, however, did use intentional interfaces to move between multiple perspectives in his own design sessions. This member used perspectives to help in learning the system. He moved between canvases containing large diagrams and others containing screenshots of the user interface in order to understand how the different pieces worked together. 

In the case of the interaction design group, it was difficult to judge their performance with the fifth design behavior because they had little time to experiment with intentional interfaces in comparison to the other groups. During their five day research session, they used the grid to conduct their design session, in which they used the grid extensively to produce several different perspectives and they did move between those different perspectives, sometimes even rapidly switching between several canvases. During their one hour brief session in which they used the version of Calico with intentional interfaces in order to create storyboards, they limited their session to a single canvas, and ignored the intentional interfaces feature altogether. They reported that it was important to them to be able to see their sketches in their entirety the whole time. 

While the three groups used intentional interfaces to different degrees for moving between canvases, all three groups found the ``copy canvas'' ability useful. The interaction design group in particular used a template canvas to begin explorations of new perspectives. For each canvas, they set a generic topic, such as ``Design behaviors'' (Figure \ref{fig:ixdgroup:session1:c}), from which sub-categories emerged from their grouping of content. Both the OSS group and the research group used copied content to examine elements juxtaposed against other representations, such as tables and mockups.

%- all sw groups used it in the classical sense
%- interacitn designers explored emergent dimensions of data
%- researchers used it for exploring the system.

\subsubsection{Design Behavior 6: They move from one alternative to another}

Two of the groups did use multiple canvases to explore multiple alternatives, but the interaction design group did not. Within the OSS group, the alternatives were often generated as a result of conflicting opinions during discussion. Multiple members had conflicting opinions, which inspired some members to copy a canvas and generate their own interpretation. In another case, a member of the research group returned to the same diagram of a process flow at a later point after having implemented it, and created several alternative copies of the canvas, in which he explored different process flows. Across both the OSS group and research group, the members used tags to label their canvases as alternatives. Also common in both groups, the members requested better tagging functionality to declare which alternative they ultimately chose, but also did not want to discard unused alternatives, because they valued them as records of past design explorations.

In the case of the interaction design group, they did not explore alternatives using multiple canvases. Unlike the other two groups, the goal of the interaction design group in their sessions was to build a better understanding the people they interviewed within a particular context. Instead of drawing multiple alternative sketches, they negotiated alternatives verbally, for example, by proposing different ways of categorizing content in a canvas, or if a person belonged to one category or another.

\subsubsection{Design Behavior 7: They move from one level of abstraction to another}

Similar to the role of perspectives, members from the OSS group and the research group moved between levels of abstraction to help focus their attention within the design. The OSS group moved between levels of abstraction in their sessions to step through the flow of data within their software architecture. In stepping through components, they copied canvases, expanded scraps to dive into more detail, and created new canvases with scraps that represent a high level view of their architecture. In the case of the research group, one member shifted a level of abstraction to break down diagrams and focus on relevant parts. He reported creating a copy of a process flow that depicted the entire system, erasing all components not relevant to his task, and designed a subpart of the system. In the case of the interaction design group, they instead performed a bottom-up approach, working from dozens of image scraps, and creating high level abstractions that described the people in the photos. All observed groups created canvases with text, either handwritten or using text scraps with list scraps, that summarized the contents of other canvases, which they referred back to while designing components. 

The OSS group and research group used intentional interfaces to move between levels of abstraction, and the OSS group additionally used scraps for this purpose as well. The OSS group used it the most fluidly, where they used intentional interfaces to continue their work onto another canvas without interruption. When ``jumping into'' a component, they copied a canvas, tagged the canvas using intentional interfaces, and reused the scraps that were copied from the previous canvas.  The OSS group used scraps to represent the components that they ``stepped into'' and ``stepped out of''. When ``jumping out of'' a component, they created a blank canvas and sketched the component. The research group did not use intentional interfaces as often, but reported it as useful for moving abstractions, and the interaction design group did not use it at all for this purpose, neither in their sessions with the grid nor their session with intentional interfaces.

%- yes, true for oss group and researchers
%- requires a certain amount of complexity

\subsubsection{Design Behavior 8: They perform mental simulations}

All groups reported that they mentally stepped through their sketches, both verbally in groups and on their own. Both the OSS group and the research group mentally simulated the flow of data within their system. The OSS group displayed sketches of their architecture on the large electronic whiteboard with Calico in meetings, and discussed the same sketch for long periods of time, both gesturing at components with their hands and using the fading highlighter from a tablet that was remotely connected to the same canvas. The research group performed similar activities, but rather than drawing entities and software components, they used process flows, and walked through the states of data as it traveled through their system. The interaction design group, instead, walked through user scenarios, and sketched flow diagrams that described the story of the user. The interaction design group also mentally walked through the stories of the people they interviewed by sketching elaborate work flows, such as in Figure \ref{fig:ixdgroup:session1:f}.

The OSS group made made heavy use of the fading highlighter in this activity, the research group used it some. In one particular meeting, four members of the OSS group discussed a Calico sketch for 30 minutes using the fading highlighter, as depicted in Figure \ref{fig:designbehaviors:mental-simulation-fig-b}. The fading highlighter allowed members to use a tablet to mark up the sketch on the large electronic whiteboard from their seat, allowing them to circle items as in Figure \ref{fig:designbehaviors:mental-simulation-fig-a}.  The research group also used the fading highlighter in their meetings, but not as often. They reported that they internalized their process flow diagrams, and did not need to perform as many detailed walkthroughs of their design. The fading highlighter was reported as being useful when a person had a tablet in their hands, and the screen was broadcast to everyone else. They also stated that they would have used the fading highlighter more in their walkthroughs had they remembered that the fading highlighter existed.

\begin{figure}%
  \centering
  \subfigure[Example of single fading highlighter stroke] {
      \label{fig:designbehaviors:mental-simulation-fig-a}
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/designbehaviors/mentalsimulation}
   }
  \subfigure[Composite of 10 minutes of fading highlighter use] {
      \label{fig:designbehaviors:mental-simulation-fig-b}
      \includegraphics[width=7.5cm,keepaspectratio]{./figures/Evaluation/designbehaviors/mentalsimulation-composite}
   }
   \caption {The OSS group sometimes spent as long as 30 minutes walking through the data flow of a diagram using the fading highlighter.}
   \label{fig:designbehaviors:mental-simulation-fig}   
\end{figure}%

In contrast to the other groups, the interaction design group did not use the fading highlighter. There are two possible reasons for this. First, the fading highlighter was most useful when using multiple devices, but the interaction design group preferred to use only the electronic whiteboard. Second, it is possible that the interaction design group preferred to sketch the mental simulation as a workflow instead of using the fading highlighter. The members of the interaction design group were both skilled in sketching visual icons expressing ideas in walkthroughs, and could readily create these icons without interrupting their work flow. Figure \ref{fig:ixdgroup:session1:f} depicts an example sketch that includes such icons.

\subsubsection{Design Behavior 9: They juxtapose sketches}

While not performed often, the three groups did occasionally set their representations side-by-side within the same canvas, i.e., \textit{juxtapose} them. In the majority of sketches created, the users preferred to create multiple canvases, and compared content informally by moving back and forth between canvases. However, they did juxtapose content within the same canvas for specific purposes, such as early exploration and reference.

The OSS group and the research group used scraps to help them juxtapose several different perspectives at once during very early phase exploration. In creating a new user interface, the OSS group created lists, box-and-arrow structures, and low-fidelity interface mockups within the same space, as seen in Figure \ref{fig:ossgroup:session1:a}. During this early phase brainstorming, the box-and-arrow diagrams evoked imagery of possible user interfaces in the OSS group members, which they sketched out in the periphery using scraps. The OSS group moved the content around the canvas using scraps, and also placed the objects side-by-side. The research group similarly used mockups alongside software structures, but instead began from the user interface and wrote pieces of code. 

The OSS group and research group also juxtaposed sketches in order to have a reference while creating a newer sketch. One OSS group member imported source code as an image to reference while designing psuedo code. A research group member also imported screenshots of the application's user interface while designing his part of the system. He further copied pieces of the process flow and used an adjacent table to step through the diagram, as in Figure \ref{fig:researchgroup:c}.

The interaction design group also juxtaposed their sketches, but did not use scraps to do so. They created multiple charts within the same space, finding that sometimes their data did not fall within one type of categorization, but two different types, as in Figure \ref{fig:ixdgroup:session1:f}. 

\subsubsection{Design Behavior 10: They review their progress}

Evidence that users reviewed their progress during their design sessions was more difficult to distinguish, but all groups reported performing this activity to some degree. Nearly all participants summarized or distilled the essence of an activity to a bullet point list in a canvas, which they would occasionally reference and update. The OSS group seldom marked up canvases with annotations, but instead created canvases with lists, in which they would note details about software components and entities from their past conversations. The research group would sometimes create bullet point lists to guide their meetings, but most often turned to Google Docs to review their agenda within their meetings. The interaction design group used a single canvas with several notes from their interviews, to which they referred when moving between different perspectives. 

Intentional interfaces served a strong supporting role when reviewing progress in all groups. Usage logs showed that members from all groups would occasionally rapidly move back and forth between several canvases, or move to a bird's eye view to review several canvases at once. Interviews confirmed that they were reviewing progress across canvases and moved to this view to ``take it all in'', respectively. All groups reported that the cluster view was not detailed enough to compare components, but instead it served to anchor discussions and allow those present at meetings to gesture at the canvases while they talked.

\subsubsection{Design Behavior 11: They retreat to previous ideas}

Retreating to previous ideas was a behavior that was only observed in multi-week, long-term design sessions. Both the OSS group and the research group  continued to use Calico within the same project, and reported that they did not return to previous ideas until a later design session. Members from both the OSS group and the research groups reported that they created new alternative designs in later sessions, but returned to past sessions to refresh their memory on the past approaches that they took. In the case of the research group, they performed a significant restructuring of their software architecture, but one of the members reported returning to out-of-date sketches because it reminded him of the rationale for certain design decisions.

Intentional interfaces supported the OSS group and research group in retreating to previous ideas. The OSS group very rarely returned to past sketches, but they reported that although infrequent, they found it useful on occasion. The research group reported that intentional interfaces was useful to them in returning to designs that were several months old. The research group stated that they could identify their old design sessions from the cluster view based on the visual structure of canvases. The chaining of their canvases in the cluster view provided contextual cues, such as relationship of canvases and order of canvases in chains, that helped the research group members remember the meaning of their sketches. 

\subsection{How they collaborate on them}

\subsubsection{Design Behavior 12: They switch between synchronous and asynchronous work}

The OSS group and the research group switched between synchronous and asynchronous work in their meetings, while the interaction design group did not. During their respective meetings, members from the OSS group and the research group deviated from the ongoing discussion to their own canvas to explore an opportunistic thought, and later share their insight with the group. The interaction design group, in contrast, have an established work culture of working in pairs to brainstorm and reflect on each others' work. This led to the interaction design group not exhibiting this behavior.

Members from the OSS group reported that they deviated to asynchronous work at least once every design session, and did so using the intentional interfaces feature. 
In one instance, a member from the OSS group disagreed with the layout of a user interface in Figure \ref{fig:ossgroup:session1:c}, and used the ``copy canvas'' feature to begin the process of creating the alternative in Figure \ref{fig:ossgroup:session1:d}. After creating his alternative, the member presented his changes back to the group on the large electronic whiteboard. In another instance, another member of the OSS group wanted to unify the diagrams diffused across several canvases, and deviated from the group discussion to draw the system in a specific use case, which is depicted in Figure \ref{fig:ossgroup:session2:h}. Once the member brought this use case of the group's attention, they created several copies of this canvas to examine use cases with different inputs. In both of these examples, intentional interfaces helped the group members navigate between these canvases by chaining them.

The research group also engaged in this design behavior using intentional interfaces, but to a lesser degree than the OSS group. In their design sessions, they navigated to other canvases in order to reference past sketches, and also to update sketches in other canvases. The members of this group sometimes requested the other group members to join them in another canvas, but the remote member reported difficulty in navigating to that canvas using intentional interfaces. In contrast to the OSS group, the research group did not report branching off to create alternatives. This was possibly due to the setting. Members in this group did not have as much face-to-face time as members of the OSS group who shared an office, which led to members focusing more in meetings. The members of the research group met regularly one to two times a week, which they used to update each other on their progress, and discuss issues they encountered. 

The interaction design group did not engage in this design behavior. During their sessions, one member of the group controlled the board while they worked together to address the design problem. One member of the group took the lead in creating sketches, while the other offered his feedback verbally.

%- an enabler
%- most important in oss group
%- ixd didn't do it at all
%- res less so
%
%- true in group design sessions

\subsubsection{Design Behavior 13: They explain their sketches to each other}

All groups encountered situations in which they needed to explain their sketches to one another. The members from the interaction design group worked very tightly together, and most explanations came from one designer challenging the design decisions of the other designer. Members of the OSS group worked more independently, where some used Calico on their own, and later presented their designs to receive feedback from other developers in the group. The members of the research group operated much more independently, in which several days would elapse before they coordinated their efforts and would present summaries of their latest work to one another. In nearly all cases, explanations were primarily carried out by pointing, gesturing in the air, and freely speaking to one another.

The highlighter was the most helpful feature to support this design behavior, however, most teams reported often forgetting to use it ``in the heat of the moment''. The interaction design group did not use the fading highlighter, preferring to simply speak out loud. The OSS group used the fading highlighter for extended periods of time when stepping through an explanation of a design that spanned several canvases. They reported sometimes being slowed down during explanations when moving between canvases, requiring that they announce what canvas they move to so that everyone else can move to that canvas as well. The research group also found the highlighter helpful, but reported often forgetting that the feature existed.

\subsubsection{Design Behavior 14: They bring their work together}

Groups very rarely brought their work together, or merged their work from separate canvases, after performing asynchronous work. The interaction design group did not perform asynchronous work and did not have an opportunity to exhibit this behavior. The research group used Calico to present their work in weekly meetings, but they did not combine their work. The OSS group, however, did bring together separate ideas, but they did so by creating new canvases. 

The OSS group created new canvases to combine work using intentional interfaces. In two separate occasions in which members from the OSS group moved between synchronous and asynchronous work in the same meeting, they generated copies of the same canvas that were variations of one another, but they did not merge these canvases. Instead, they created a new canvas, linking it to the previous canvases using tagging, and sketched a use case scenario that summarized the content from previous canvases. 

%- not so much in the real world... everything that is sketched are suggestions. Bringing work together is too much work. They may summarize though.

\section{Summary}
\label{chapter:evaluation:summary}

In this chapter, I presented my observations of Calico Version Two in use at three locations. The commercial open source software company provided insight in how a software team may use Calico to support ongoing with work among developers while coding. The interaction design group provided insight into how interaction designers may use Calico to help them create personas and storyboards. Lastly, the research group provided insight into how a set of distributed researchers may use Calico in support of long-term collaboration in the development of a software system for a period of several months. In the next section, I bring these separate uses together by discussing their collective lessons and insights in the context of Calico's support for design behaviors.

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
%
%\subsubsection{Design Behavior 1: They draw different kinds of diagrams}
%
%\subsubsection{Design Behavior 2: They produce sketches that draw what they need, and no more}
%
%\subsubsection{Design Behavior 3: They refine and evolve their sketches over time}
%
%\subsubsection{Design Behavior 4: They use impromptu notations}
%
%\subsubsection{Design Behavior 5: They move from one perspective to another}
%
%\subsubsection{Design Behavior 6: They move from one alternative to another}
%
%\subsubsection{Design Behavior 7: They move from one level of abstraction to another}
%
%\subsubsection{Design Behavior 8: They perform mental simulations}
%
%\subsubsection{Design Behavior 9: They juxtapose sketches}
%
%\subsubsection{Design Behavior 10: They review their progress}
%
%\subsubsection{Design Behavior 11: They retreat to previous ideas}
%
%\subsubsection{Design Behavior 12: They switch between synchronous and asynchronous work}
%
%\subsubsection{Design Behavior 13: They explain their sketches to each other}
%
%\subsubsection{Design Behavior 14: They bring their work together}
 \newpage 
 \newpage \chapter{Discussion}
\label{chapter:discussion}

In the previous chapter, I reported on the experiences of three different groups using Calico. In this chapter, I bring these observations together to answer the research questions posed in Chapter \ref{chapter:research-question}.

The rest of the chapter is organized as follows. In Section \ref{discussion:strengths-and-weaknesses}, I review the strengths and weaknesses of each feature, drawing upon how they were used across all three field evaluations. Section \ref{discussion:cog-dim}, takes a step back from the observations and examines the theoretical potential of Calico's features by performing a cognitive dimensions analysis, revealing insight into why features did and not work. Section \ref{discussion:overall-strengths-weaknesses} then examines the overall strengths and weaknesses of Calico, examining the benefits and drawbacks of using Calico in a work environment. Section \ref{discussion:interviews} discusses the feedback from the users, their personal experiences, and what they saw as important in their experiences. Section \ref{discussion:summary} then summarizes the contributions of this chapter.

\section{Strengths and weaknesses of each feature}
\label{discussion:strengths-and-weaknesses}

In this section, I examine each feature on its own merits to discuss their strengths and weaknesses. Table \ref{chapter:discussion:strengths-weaknesses} provides an initial summary of this section, and each feature is examined in detail in the subsequent subsections.

\begin{center}
\begin{longtable}{|p{3cm}|p{6cm}|p{6cm}|}
\caption{The set of design behaviors and the features that supported them}\\
\hline
\textbf{Feature} & \textbf{Strengths} & \textbf{Weaknesses}\\
\hline
\endfirsthead
\multicolumn{3}{c}%
{\tablename\ \thetable\ -- \textit{Continued from previous page}} \\
\hline
\textbf{Feature} & \textbf{Strengths} & \textbf{Weaknesses}\\
\hline
\endhead
\hline \multicolumn{3}{r}{\textit{Continued on next page}} \\
\endfoot
\hline
\endlastfoot
\hline
Scraps \& 

connectors &
%strengths
1. Flexible nature is useful in creating many representations relevant to software

2. Gestures enable quick manipulation of hand drawn sketches and scraps without mode switching

3. Existing hand-drawn boxes could be refined into scraps

4. Bubble menu is self discoverable
 &
%weaknesses

1. Requires training to use properly

2. The press-and-hold gesture for select and moving scraps was slow for continuous arranging of a large number of scraps

3. Could not change scrap color or border, text scraps were ``visually heavy''

   \\
\hline
Palette &
%good 

1. Bootstrapped design sessions by importing existing artifacts into multiple canvases

2. Used to build set of reusable icons in storyboards

3. Used as a global clipboard to copy content across canvases in order to juxtapose them

&
%bad 

1. Hard to find scraps in palette when it has many items (more than twenty)

2. Users could not add plain sketches to the palette, must be turned into a scrap first

\\
\hline
Intentional 

interfaces &

1. Clusters provided a simple metaphor to separate content of different projects

2. Linking canvases allows a narrative to be constructed from a set of canvases

3. Made free space to sketch immediately available because they can immediately jump to new canvas or copy a previous one

&
%bad
1. Moving between the canvas view and the cluster view was not a smooth action, canvases were too small to visually distinguish after moving

2. Users had a difficult time understanding where a new canvas appeared in the cluster view

3. Difficult to juxtapose content across canvases

 \\
\hline
Fading 

highlighter &
%good
1. Supported explaining sketches and mental simulations

2. Unlike traditional pointers, fading stroke allows for transient annotations such as arrows, underling, circling, etc.

&
%bad 
1. Strokes made with highlighter are anonymous

\label{chapter:discussion:strengths-weaknesses}
\end{longtable}
\end{center}

\subsection{Scraps}
Scraps, on the whole, were a relatively successful feature in Calico, and a significant improvement of their Calico Version One counterpart. Scraps replaced the lasso functionality to manipulate sketches, which enabled users to manipulate content without changing modes. Scraps also depicted software representations such as software components, process flows, and user interfaces. They sometimes were used to represent lists, though not often. While there are opportunities to provide better support, such as the ability to change its fill and border color, scraps saw a significant amount of use across all groups. 

A strong quality of scraps was their flexibility in representing diagrams. While scraps were not always used in favor of regular whiteboard sketching, they were used often and yielded benefits such as helping diagrams evolve more gracefully and arranging content. Actions which become natural when using scraps, such as annotating it with a color patch, underline, are not straightforward in other mediums. The same actions could be done with drawing tools, but arranging sketches would be more complex. Alternatively, one could design using a more formal tool which would allow for easy arranging of sketches, but the tool may not be able to freely add annotations or symbols. The flexibility afforded by scraps led to the unexpected behaviors in the experiment, such as when the interaction design group tagged them using color and the research group color coded their scraps as well. In comparison to Calico Version One, which saw a relatively narrow set of representations created with scraps in experiments, Calico Version Two yielded a much greater variety of representations created with scraps. It is possible that the longer term evaluation led to more time to become accustomed to using scraps; however, it is more likely that the revised functionality of scraps led to this improvement.

What scraps lacked, however, was more expressive power in their visual features. Users could not change the blue color of scraps and their outline, which many users requested the ability to do. Users in the studies commented that they wished to create text scraps without the blue boarder, as they felt that it made the sketch ``visually busy''. In another session, a user wanted to create text scraps with text of different colors in order to encode meaning, but instead used hand-written text in order to use colors. While these and other features may have been useful, their absence did not obstruct the sessions as users found ways around them, and users remarked that these sessions were provisional.

Scraps gestures, such as creating, selecting, and moving, were a strong and weak point for scraps, depending on the situation. Particularly with long-term users, scrap gestures became a powerful tool to quickly move and copy content. When working with box-and-arrow diagrams for software, they allowed content to be rearranged quickly. When working with lists, they allowed individuals to adjust spacing in handwritten text or make more space. In user interfaces, they enabled rearranging of elements to simulate using the sketched interfaces. Unfortunately when used to categorize large number of scraps, as the interaction designers did, the press-and-hold gesture to select and move scraps were perceived as too slow. The interaction designers, in this case, requested a separate mode specifically targeted at moving scraps.

A weakness of scrap gestures was that they required some practice to become fluid. Gestures such as the landing zone to create scraps were enabled quick manipulation, but new users required some training before they could trigger the gesture reliably. 

Scrap gestures in Calico Version Two, however, was a large improvement on the scrap gestures in Calico Version One, which were not discoverable and were reported as unpredictable. In contrast, the bubble menu in Calico Version Two was reported as being straightforward and discoverable. Further, the landing zone used to create scraps was self discovered by users, while the equivalent action in Calico Version One was also difficult to discover. 

Lastly, the ability to convert regular hand-drawn sketches into scraps, i.e., refine them, further benefited teams. This addressed a significant issue in Calico Version One, which was that individuals often did not use scraps because they would first hand draw a box, and later draw a scrap around the box, resulting with a hand-drawn box inside a scrap. While not used often, users at field sites did use it for both refining existing sketches into scraps and as a recovery mechanism when they failed to trigger the ``landing zone'' gesture correctly.  Users further refined their scraps by using list scraps, which organized scraps into a linear compact list. 

\subsection{Palette}

While the palette significantly improved on the palette of Calico Version One, it still remained one of the weaker features in Calico Version Two. This was due most in part because the palette did not noticeably support impromtu notations for software oriented groups. However, while not performing strongly in this specific design behavior, the palette succeeded in other aspects, such as supporting impromtu notations for storyboards and acting as a global clipboard to transfer content between canvases.

The palette performed strongly in supporting users in three scenarios. First, the palette served to bootstrap a design session for interaction designers by importing several dozen images from outside of Calico. The imported images drove the design session which spanned several days. Second, the palette actually did serve to store a set of graphical icons that were improvised by the interaction design group in their design sessions. Members from the interaction design group found these icons useful in creating storyboards. Third, all groups used the palette to juxtapose new sketches against previously created ones. They did so by using the palette as a global clipboard to copy old sketches to the canvas with the newly created sketch.

Some aspects of the palette could have used further improvement. Users had a difficult time locating items in the palette when it contain in excess of twenty scraps. The interaction designers imported dozens of images of faces, which were difficult to distinguish because of their small size in the palette. Also, users wished to add sketches to the palette without first making the sketch into a scrap. Users reported that making the sketch into a scrap made it too heavy, e.g., they simply wanted the sketch without a scrap's blue background and border.

A potential weakness of the palette was that users designing software systems did not use it to perform the fourth design behavior, inventing impromptu notations. However, this was not a weakness of the palette, but instead caused by two factors: 1) users found it faster to redraw certain sketches because it was faster, and 2) other Calico features were redundant with the palette. With respect to the first point, users often only needed to sketch the name of a component, which could be created much more quickly manually using scraps than finding the component in the palette. With respect to the second, the availability of the copy functionality in numerous forms lessened the need for the palette. For example, when users sketching a software system wanted to create an alternative or move perspectives, they often used the ``copy canvas'' button in intentional interfaces, or used the ``copy scrap'' button in a scrap's bubble menu.

\subsection{Intentional interfaces}

Intentional interfaces was a relatively successful feature in Calico. It advanced on the concept of the grid from Calico Version One, and satisfied numerous design behaviors in doing so. It provided qualities that were not available in the grid interface, such as names for canvases, relationships, and order of work performed. Intentional interfaces, however, had some weak points as well. Navigation was sometimes clumsy in the cluster view, juxtaposing sketches across canvases was difficult, and understanding the relationships between canvases was sometimes difficult for new users.

Many of the grouping mechanisms in intentional interfaces improved upon the grid interface of Calico Version One. While the grid was well received by users of Calico Version One, users increasingly desired ways to separate and categorize their content. Intentional interfaces addressed this problem in several steps. First, clusters provided a generic method to divide canvases into topics, which helped partition content between different projects in Calico.

Second, linking canvases provided another level of organization of canvases, while at the same time offering an ordering to the canvases. Providing an ordering to the canvases was shown to be beneficial in practice as it allowed a set of canvases to compose a narrative. A sequence of canvases could show a walkthrough of a software architecture as well as the exploration of the design space across multiple perspectives, abstractions, and alternatives. Further, grouping canvases by linking them helps users better recall design sessions and understand the content of their sketches several months afterwards.

A third benefit of intentional interfaces was the immediateness of free space. In comparison to the regular whiteboard, users felt more at ease in creating more sketches because they could always create more space and return to previous sketches. The presence of both the ``new canvas'' and ``copy canvas'' buttons made moving to a new sketch a ready-at-hand action, which they used to explore different perspectives and new alternatives.

Users encountered a few issues using intentional interfaces as well. First, the movement between the canvas and the cluster view was not a smooth transition for the user. When loading the cluster view, the perspective always zoomed out, which made the individual canvases too small to distinguish. Second, users had a difficult time understanding the shape of the cluster when creating new canvases from within a canvas itself. One member of the OSS group described the shape of the cluster as ``wizardry'', in which it was a mystery to him as to where new canvases would appear. Third, users could not easily compare contents between canvases.

\subsection{Fading highlighter}

	
The highlighter feature worked well for a particular set of scenarios. The fading highlighter stands on its own as users did not combine its use with other features, but it was useful in scenarios that the other features did not support. Particularly, it supported users in mentally simulating over their work, and also in explaining concepts to others. 

The second strength of the fading highlighter was that it enabled transient annotations in verbal walkthroughs and explanations. Traditionally, verbal explanations of sketches include either a simple pointer, like a baton or laser pointer, or the speaker must permanently mark up a sketch. The fading highlighter provides a compromise of the two approaches, allowing the speaker to use symbols that convey more meaning, such as drawing arrows between sketches, underling them, and circling key points for emphasis.

While the fading highlighter addressed these situations well, it could still be improved to provide better support the two design behaviors that it targets. First, it could provide better support for explaining sketches by providing better awareness features. For example, the strokes could somehow show who is making the stroke. Second, mental simulations could be better supported if the sequence of strokes were somehow recorded. Many of the mental simulations explain how data is moved between components. If this explanation could be captured, and later played back, particularly with audio explanations, the design rational within the design sessions could be played back.

\subsection{Summary}

Each feature brought forward a set of unique advantages not available in the other features. Scraps were strong in supporting the kinds of sketches designers drew. The palette acted as a global clipboard and made sketches reusable across canvases. Intentional interfaces provided many ways to organize, partition, and make free space for sketches. The fading highlighter supported verbal explanations of sketches well. While each feature had usability issues, such as navigating in intentional interfaces and scraps requiring training to use properly, the features provided a net benefit to their users.

\section{Cognitive dimensions analysis}
\label{discussion:cog-dim}

Having examined the use of Calico's features \textit{in practice}, this section now examines the \textit{theoretical} support of those features by performing a cognitive dimensions analysis. The cognitive dimensions framework exposes the affordances that a notation or medium offers. Analyzing Calico's features through this framework provides a basis to determine how those features help or hinder the notation that is implied within Calico. In order to perform the cognitive dimensions analysis, I look at each dimension, and discuss factors that either increase, decrease, or, all things being equal, provide equivalent support for designers as compared to the whiteboard. These findings are summarized in Table \ref{table:discussion:cognitivedimensions}.

%Cognitive dimensions is the golden standard of what Calico should be. Is it possible.. not if they did it. CD exposes to what uses a notation or tool leads.

%Table \ref{table:discussion:cognitivedimensions} summarizes the findings in this section. In most cases, Calico's features provided benefits to one or more cognitive dimensions. Intentional interfaces played a large role in finding relevent canvases in designs diffused across canvases. Scraps and the palette made sketches less viscious and easier to manipulate. Overall, Calico improved on the regular whiteboard for the following dimensions: abstraction, hard mental operations, premature commitment, provisionality, secondary notation, viscosity, and visibility. Calico also had some shortcomings for some of the groups. These can be attributed to inefficient support for the cognitive dimensions of consistency and error-proneness. Support for all other dimensions, including closeness of mapping, diffuseness, hidden dependencies, progressive evaluation, and role expressiveness received moderate support.

\begin{center}
\begin{longtable}{|p{3cm}|p{4cm}|p{4cm}|p{4cm}|}
\caption{\DIFdelbegin \DIFdel{What }\DIFdelend CDs Analysis \DIFdelbegin \DIFdel{Highlights about }\DIFdelend \DIFaddbegin \DIFadd{of }\DIFaddend Calico}\\
\hline
\textbf{Cognitive Dimension} & \textbf{Factors that cause an increase over the whiteboard}& \textbf{Factors that are equal to the whiteboard}& \textbf{Factors that cause a decrease to the whiteboard}\\
\hline
\endfirsthead
\multicolumn{4}{c}%
{\tablename\ \thetable\ -- \textit{Continued from previous page}} \\
\hline
\textbf{Cognitive Dimension} & \textbf{Factors that cause an increase over the whiteboard}& \textbf{Factors that are equal to the whiteboard}& \textbf{Factors that cause a decrease to the whiteboard}\\
\hline
\endhead
\hline \multicolumn{4}{r}{\textit{Continued on next page}} \\
\endfoot
\hline
\endlastfoot
Abstraction	
& %increase
Scraps help visualize abstractions and make them manipulatable
& %equal
You can draw anything
& %decrease

\\
\hline
Closeness of 

mapping	
& %increase
Can manipulate notations at the level of scraps and connectors
& %equal
Can draw in any notation
& %decrease
Annotations, like cardinality, do not move along with connectors
\\
\hline
Consistency	
& %increase
Can save and refer back to previous sketches, on whiteboard they may have been erased
& %equal
Must be socially enforced
& %decrease
Difficult to socially enforce when content is diffuse and requires hard mental operations, as opposed to whiteboard where content is all in one space
\\
\hline
Diffuseness	
& %increase
Calico breaks up the whiteboard space across several canvases
& %equal
Small designs stay on one board
& %decrease

\\
\hline
Error-proneness	
& %increase

& %equal
No error checking
& %decrease
Manipulating and automated copying of existing sketches may prevent errors caused by redrawing sketches
\\
\hline
Hard mental operations	
& %increase
Design may be diffuse in Calico and details of sketches scattered across many canvases
& %equal
Small designs are contained within one area
& %decrease
Content can be imported into Calico using image scraps; content can be positioned side-by-side using scraps; palettes allow users to copy content for reference across canvases; navigation button allows user to return to previous content quickly
\\
\hline
Hidden dependencies	
& %increase
Content may be diffuse, making hidden dependencies hard to find
& %equal
Small designs have all potential references in one area
& %decrease
Intentional interfaces helps manage diffuseness by grouping content together to make hidden dependencies easier to find
\\
\hline
Premature commitment	
& %increase

& %equal
Medium is considered provisional; rejected decisions can be preserved by crossing them out rather than erasingor deleting them
& %decrease
Easier to explore alternatives by copying existing ones using scraps, palette, and intentional interfaces
\\
\hline
Progressive evaluation	
& %increase
All sketches are preserved and can be returned to; fading highlighter allows evaluting without marking the original sketch
& %equal
Users must manually judge the ``rightness'' of a sketch by manually reviewing it
& %decrease
Diffuse designs are harder to progressively evaluate; fading highlighter marks during an evaluation cannot be later referred to as regular marks could be on a whiteboard
\\
\hline
Provisionality	
& %increase
Users can make copies and explore alternatives without losing original ideas; more sketches can be reviewed across canvases in intentional interfaces; fading highlighter allows sketching over design without permanently marking it; canvases can be tagged as alternative using intentional interfaces
& %equal
Medium is considered provisional by users; anything can be changed at any time
& %decrease
More difficult to evalute sketches diffused across canvases; marks from progressive evaluation no longer remain and cannot be referenced
\\
\hline
Role expressiveness	
& %increase
User has unlimited space to fully express all parts of a design
& %equal
User is limited by the space available
& %decrease
Designs spread across canvases have higher diffusion, lower visibility, and may require hard mental operations to reference other canvases
\\
\hline
Secondary notation	
& %increase
Second notations can be attached to scraps and move with scraps; custom shape of scrap can be used; color of connectors can be used to capture meaning
& %equal
Users can sketch in any secondary notation
& %decrease
Annotations outside of a scrap do move with a scrap and the annotation is left behind
\\
\hline
Viscosity	
& %increase

& %equal
Users can sketch anything they want; a full whiteboard is less likely to be changed because it contains many important decisions
& %decrease
Scraps make sketched content not ``locked in'' such that can easily be moved and manipulated; gestures make manipulating scraps a fluid action
\\
\hline
Visibility	
& %increase

& %equal
Sufficiently small designs are entirely visible on the whiteboard
& %decrease
Designs diffused across multiple canvases have very low visibility, leading to hard mental operations
\label{table:discussion:cognitivedimensions}
\end{longtable}
\end{center}

\subsection{Abstraction}
Abstraction refers to Calico's ability to support representations at different levels of abstraction. Since Calico at its most basic supports plain sketching, it inherently allows maximum flexibility in what can be represented, much the same as the traditional whiteboard. 

Calico, however, improves on the regular whiteboard because users can manipulate those abstractions at the level of scraps or, when using intentional interfaces, entire canvases. On the whiteboard, the designer can only draw and erase strokes. In Calico, they have scraps to depict abstractions, which support quick movement of the drawn abstractions, implicit grouping by stacking scraps, and have arrows that remain attached to the scrap as it is moved. Intentional interfaces enables the user to explore different levels of abstraction across multiple canvases, and supports the user in navigating between these levels of abstraction by linking canvases. 

When the content of a design session is suffuciently small, the whiteboard may have a better expereince than Calico because all abstractions are available in one space, while in Calico they may be diffused across several canvases. When on a single whiteboard, the designer can immediately look at all levels of abstraction at once. If the content of the design grows beyond what can be sketched on the whiteboard, then Calico becomes more useful, as it can capture all sketched content across several canvases, whereas on the whiteboard that content would need to be erased, or scattered across multiple whiteboards, posters, or pages. 

\subsection{Closeness of Mapping}
%You can draw in any notation. Factors that increasing is that you can manipulate notations at the level of scraps and connectors. factors that decrease it is annotations like cardinality... they don't move along with connectors.

%Very similar to the whiteboard. Compared tot he whiteboard, all that is preserved. Now I've given people the way to manipulate them as wel.

Closeness of mapping refers to how closely representations map to the referred concepts. Calico is similar to the whiteboard in this regard in that designers are at liberty to sketch any representation. If the designers decide that there is a more suitable representation, they are able to create a new sketch that depicts that representation simply by drawing its corresponding shapes. These shapes, then, can be turned into scraps when so desired. 

When using scraps, Calico has the positive consequence that representations can be manipulated as objects, which may more closely map to the mental model of the designer than plain strokes would. Further, where on the whiteboard representations may need to be erased and redrawn to create a sketch that closely matches the desired notation, scraps are reused by copying them. 

A negative consequence of using scraps is that certain visual visual annotations, do not move along with scraaps and connectors. This may cause sketches to become less closely mapped to their concept as cardinality annotations may become disassociated from their target sketch. This may cause additional work as users will need to manually move annotations with scraps, or simply leave the annotation behind after the scrap is moved. 

%
%- people could get very close to what they are representing. Importing screenshots into the environment is a big one.
%- interaction designs could manipulate their objects directly, no indirect names
%- software people could import snippets of code
%- researchers imported screenshots of user interfaces, source code, etc.

\subsection{Consistency}
Consistency refers to the degree to which features of structures and syntax are used the same way throughout. Calico does not provide an automated method to enforce consistency. As with sketching, maintaining consistency between elements and sketches is up to the users to ``socially'' enforce \cite{Petre2013BookChapter}. If all content can be condensed to a single whiteboard, the designer can refer back to previous sketches to check for consistency. On Calico, since the designer is encouraged to use multiple canvases, it may be more difficult to manually verify consistency across all canvases.

However, if the design is too large for a single whiteboard, Calico would provide some help over the whiteboard. Where on the whiteboard sketches would need to be erased, the designer may save them in Calico, and refer back to them to verify consistency between old sketches and new.

%- consistency brought by calico comes from copying, none beyond other, unless enforced by personal discipline.
%- palette supported this, but seldom used this way
%- multiple perspectives enforced consistency,
%- researchers used multiple documents, hard to maintain consistency across those

\subsection{Diffuseness}

%There's a cost with Calico - it makes things more diffuse. By taking the original whiteboard, cutting it into pieces, I've actually made it a little bit harder (speaking of Calico). 
%
%Counter argument is that people work with it different. They partition it differently. The jury is out. 
%
%Can't say there's moderate support for diffusement, instead say there's moderate support for reducing diffuseness.
%
%I don't have as much as much space in an individual Calico canvas as the whiteboard. If everything can be sketched on the whiteboard, it's superior because I can see everything at once. However, if the design grows beyond the space of a whiteboard, diffuseness becomes obliviousness on the whiteboard because what is not immediately visible has been erased. Calico incrementally becomes more important because diffuseness on the whiteboard has become obliviousness, sketches are no longer exist, but in Calico they do. However, in Calico there is now the navigation problem among these things. The structure of intentional interfaces is one way we address that. It is unclear... it doesn't erase diffuseness, it doesn't moderately support it, but what it does do is acknowledge it, and we have built features to address that. More research is required to determine how well the features combat diffuseness, the support is there.

Diffuseness refers to how much the meaning of sketches is spread out across multiple sketches. Calico potentially increases diffuseness because the potential for an unlimited number of canvases may encourage designers to break up their designs across multiple canvases. On the whiteboard, designers are forced to limit their design sketches to the limits of the whiteboard.  If everything can be sketched on the whiteboard, then it is superior because designers can see everything at once. 

However, if the design grows beyond the space of a whiteboard, Calico's functionality helps in dealing with diffuseness. On the whiteboard, diffuseness becomes obliviousness because what is not immediately visible has been erased. Calico incrementally becomes more important because, while on the whiteboard sketches no longer exist because they have been erased, in Calico they remain. Yet, this raises a new issue, which is the problem of how to navigate between sketches in Calico. The structure of intentional interfaces is one approach to managing the diffuseness. It is unclear if intentional interfaces effectively addresses diffusenss, as it does not remove it. What it does do is acknowledge it, and gives the user tools to work around manage it themselves. More research is required to determine how well the features combat the consequences of increased diffuseness.

%Further, enabling one to copy canvases and transfer content using the palette encourages diagrams to be diffuse. However, intentional interfaces mitigates these issues by helping to navigate designs diffused across different canvases. By providing links between those  canvases, users can more easily find content immediately relevant to the current canvas. Intentional interfaces does not provide a fully comprehensive solution to diffused diagrams, but it does moderately support it in a lightweight fashion.

\subsection{Error-proneness}

Error-proneness refers to the degree to which a notation induces ``careless mistakes'' \cite{Petre2013BookChapter}. As with plain sketching, Calico does not provide any safe guards against errors. Individuals are free to improvise and switch between notations, which may increase their tendency to perform errors if there is no clearly defined standard notation. 

While Calico does not alert the user of when they have performed an error, Calico's features do automate certain actions which may reduce the likelihood of errors. Scraps and connectors, for example, provide flexibility in creating representations by enabling the moving, resizing, and rotating existing sketches. They maintain the structure of objects as they are moved, which may help when working with box-and-arrow diagrams. In comparison to plain sketching, scraps and connectors help prevent errors by making additions to representations, like process flow diagrams, less tedious so they do not require diagrams to be redrawn when repositioning elements, as one would have to if sketching at the whiteboard. On the other hand, designers recognize that there is value in redrawing a sketch from scratch because each time a sketch is redrawn, its contents are re-evaluated \cite{petre2009insights}, and copying existing sketches removes this opportunity.

%The research group benefited from the use of scraps, which helped avoid errors by removing much of the tedium in working with their state diagram. When working with their state diagram on the whiteboard, they reported that it had become too large to manage. After moving to Calico, they had more flexiblity in managing their space. They were able to create more space by moving text-scraps, which moved all connectors with the scrap as well and retain the shape of the diagram. Further, the flexiblity to create connectors with custom paths, as opposed to straight lines, made connectors more legible. 

%Overally, scraps and connectors helped prevent errors by making adding additions to the state diagram less tedious, and not requiring diagrams to be redrawn to reposition elements.

%No safeguards against errors, other than what was socially enforced by the designers.
%
%- repeated elements helped avoid using the wrong elements, and reviewing content in meetings helped catch errors


\subsection{Hard Mental Operations}
%By making it electronic, and breaking it into pieces, you gain, but you lose something too. On the whiteboard, when people start to erase, you gain.

Hard mental operations refers to information that needs to be referenced or is nested. Similar to the cognitive dimension of diffuseness, in a simple or small design, both the plain whiteboard and Calico are equal in that all parts of the design are visible and there may be little need to reference outside information. However, in larger projects, Calico may have more hard mental operations because the design may be diffused across several canvases requiring the user to reference sketches outside the immediate space. Users may need to use the intentional interface feature to move between canvases to reference other sketches, or use the palette to move content so that it can be referenced in the active canvas. For a large design on a plain whiteboard, content may need to be erased, and Calico may provide better support because that content would be preserved.

Calico further reduces hard mental operations in other sutiations. It allows users to both import content and copy content to the canvas they are currently in. First, image scraps allow users to import existing artifacts into a canvas to refer to while sketching. For example, a developer may import source they are working on, or a diagram created in another tool. Second, scraps allow users to move content so that it is positioned side-by-side to what they are currently working on. Third, the palette allows users to copy content between canvases so that they can reference old content in the new space. Fourth, the navigation button in intentional interfaces allows users to return back to the most recently visited canvas. While this may still quality as needed to refer to parts of the design outside of the immediate visibility, the feature reduces the effort needed to do so.

%In the case of the OSS group, the developers needed to reference information that was spread across several canvases. Members from their team reported that they did not need to do so often all members were already familiar with the depicted system. When they did need information to be available more immediately, they copied content onto the target canvas. The developer working with source code did so by pasting screenshots of his code into his canvas. In the case of the interaction designers, they pasted pictures of people they interviewed, however they needed to reference their notes from their interviews. The researchers, however, had a greater need to perform hard mental operations because of the scale of their state diagrams. The researchers who originally developed the system eventually internalized the diagram to a degree that they no longer needed to reference it, however the researcher that needed to be onboarded created additional diagrams, such as tables, in order to help him mentally step through the state diagram. In his case, he mitigated the need to reference other canvases by copying the pieces of the state diagram that he needed into his own canvases.

%Overall, the ability to copy content within Calico somewhat reduced the need to reference other canvases. Scraps, intentional interfaces, and the palette were helpful for both importing screenshots and copying content to the immediate working area.

%How often did designers need to reference elsewhere and nest?
%
%- the OSS group had to incur hard mental operations by referencing across diagrams because of multiple levels of abstraction
%	- they moved to high levels of abstractions, and use cases
%	- hard to interpret what was there because had to keep track of everything
%- the interaction design group had to do lots of hard mental operations because interviews not represented. Also placing along axies, and mult. perspectives
%- researchers experienced a lot of hard mental operations because of the terseness of the diagrams (paths, etc.)

\subsection{Hidden Dependencies}

%On the whiteboard, everything is there. I can glance over on the whiteboard, but I can't in Calico.

Hidden dependencies refers to representations that are dependent on one another, but the dependencies are not visible because they are either not declared or not in a person's field-of-view. As observed in past studies \cite{dekel2007notation}, sketches that cross several spaces result in having dependencies on other spaces that are not explicitly stated. Sketches that begin in one canvas in Calico may extend onto another canvas, or across mediums, such as their own computer or paper, if the users are using Calico as a complementary tool. This could occur, for example, when canvases depict different perspectives or abstractions. 

The intentional interfaces feature in Calico helps in declaring and finding these hidden dependencies on other sketches by providing features that group canvases of related content. Clusters in intentional interfaces help group canvases into sets of similar topics. Linking canvases into chains provides a second level of grouping that help designers in finding related sketches. These features do not solve the issue of revealing all hidden dependencies, but they do improve working with them by by making those dependencies easier to find.

\subsection{Premature Commitment}
A notation or representation that causes a user to \textit{prematurely commit} means that it requires a person to make a decision before they have the information they need \cite{Petre2013BookChapter}. Sketches and content produced in Calico are typically used to support conversations or thinking through a design, and thus there is less of a concern to prematurely commit because all content is provisional. The sketchy appearance of content within Calico visually re-enforces the notion that content is provisional. Also, the ease of changing content, such as drawing an ``X'' symbol over rejected content, allows for decisions to be rejected without permanently discarding the idea. In this regard, Calico is equivalent to the whiteboard which also has these qualities of provisionality. 

However, Calico further reduces premature commitment in comparison to the whiteboard by making sketches more easily copied and reproduced. Features such as scraps, the palette, and intentional interfaces make it easier to explore alternatives. For example, while on a whiteboard, a user may be hesitant to erase a large figure or deviate because they do not want to lose previous work. In Calico, they can simply create a copy of it in a new canvas, or continue their work in a new blank canvas.

%Members from all groups reported not feeling pressured to commit to decisions within Calico. The OSS group reported that they valued the ideas and tasks generated from using Calico rather than the sketches themselves. They reported that they sometimes immediately revisited sketches after meetings in order experiment with alternatives before returning to their desk to implement a new change. The OSS group did email snapshots of sketches from Calico to themselves, but reported that these emails were archival and served to remind them of the ideas generated. The ideas were not considered committed until they were implemented into the system. In the research group, the members used the sketches from Calico in support of implementing their software system and writing an academic conference publication. One of the researchers remarked that nothing in Calico was permanent until ``it was written in the [academic conference] paper''. 

%Overall, the provisionality of content and the ease with which it can be changed, i.e., low viscosity, lowered the tendency for content to be prematurely committed in Calico. However, there is very little gain in this regard over the whiteboard. The ease of copying content reduces the tedium in exploring alternatives, but this was only marginally beneficial for the groups with respect to preventing premature commitment.

%Calico provided a sketching look and feel to elements. Users consciously commited to decision by converting handwritten text into rectangular scraps and sometimes formal connectors.

%- there is not a sense of commitment to what is on the board.
%- research group turned to calico/whiteboard because it gave the sense of freedom from commitment

\subsection{Progressive Evaluation}
Progressive evaluation refers to the ability to run a simulation or obtain feedback on a representation that is only partially complete in order to determine its correctness. While Calico does not directly support progressive evaluations, it does support users in manually judging the ``rightness'' of a sketch by supporting the reviewing and explanation of sketches. Much like the whiteboard, designs may be manually reviewed and discussed by close inspection. 

With large designs, Calico provides some additional benefit with intentional interfaces, which aids users in navigating between canvases to review them. However, as the sketch grows too large, it becomes progressively more difficult to manually evaluate all sketches. Further, the collaborative aspect of Calico allows remote designers to obtain feedback, during which the fading highlighter would help during group reviews to discuss content. In local designs, the fading highlighter further helps progressive evaluations by making the sketches cleaner, evaluative marks are not left behind using the fading highlighter. However, this could be considered a loss in comparison to the whiteboard as well, as designers could potentially refer back to these marks to remind them of their evaluation.

%The intentional interface feature supported the OSS group in reviewing their contents by allowing them to zoom out and switch between canvases quickly. The OSS group reported that they would switch to the cluster-view, zoom in to the canvases they had been working on, and pan while discussing them. During group meetings, they further used the fading highlighter during discussions and evaluations of components, drawing paths taken by data between components during their explanations. The researchers, similarly, engaged in verbal dialog to review their designs. The researchers engaged in review of their designs, particularly the state diagram that summarized their software system. They stepped through the state diagram to verify its correctness, and also used it in support of evaluating the completeness of their software system. They used the sketch to keep track of what was and what was not implemented, and updated their state diagram with this information.

%Overall, Calico supported individuals in performing a manual evaluation of their designs. In this regard it could be considered similar to the whiteboard, but intentional interfaces provided the benefit of reviewing multiple canvases, and also the quality that Calico was an online shared resource that was shared by everyone. 

%- all groups used intentional interfaces to review, summarized, and proceed. different perspectives

\subsection{Provisionality}
Provisionality refers to the quality that indecision or options can be expressed \cite{Petre2013BookChapter}. Given that sketching is the primary mode of operation in Calico, individuals are free to break away from formal notation to declare multiple alternatives. Users can always erase a sketch to change a value, or simply cross out a sketch so that a design can be rejected without visually removing it from the design space. Calico goes further in this supporting provisionality by supporting the quick generation of alternatives using duplication features such as scrap copying, canvas copying, and the palette. With intentional interfaces, uses can explicitly express entire canvases as being an alternative by tagging it as such within intentional interfaces.

%The OSS group used the fading highlighter to propose ideas about sketches that they were not sure of yet. After discussing a box-and-arrow diagram using the fading highlighter for more than twenty minutes, they changed elements within that diagram. They did so by drawing a large ``X'' over the component, and wrote the name of an alternative component next to the component with an ``X'' on it. In another instance, members in the OSS group used intentional interfaces to create a copy of a canvas, and tagged the canvas an ``alternative''. They did so on multiple occassions, such as during a group discussion, and also in individual design sessions. The research group expressed options in another fashion. In their case, they performed a major refactoring of their system, effectively diverging from the design in their previous sketches. Rather than deleting these old sketches, they moved to another cluster to sketch the design of their refactored system. They considered the contents of the old cluster as an archive of the first version of their system. 

%Overall, Calico provides some improvements in provisionality over the whiteboard. Within a canvas, Calico is much the same as a whiteboard in that the user has the flexibility to sketch alternative names. Across multiple canvases, intentional interfaces helps manage options at a larger scale by providing tags to label canvases as alternatives. Further, the ability to copy and create new canvases and clusters encouraged more sketching, and possibly exploration of alternatives.

%Everything in Calico was considered provisional and outside the formal specfications. Provisionality was reduced by copying content onto another medium. Teams walked away from designs, and return to them in order to be reminded.

\subsection{Role Expressiveness}

Role expressiveness refers to a reader's ability to see how the parts fit into the whole design, and how those parts relate to one another \cite{Petre2013BookChapter}. In other words, a reader can independently interpret what each part in a sketch does without further information. Given that users have the flexibility to sketch any notation they need, the limiting factor in both the regular whiteboard and Calico is the space available to create representations. With enough space, the role of each part can be better understood if all details are visible. On the whiteboard, this means that a sketch must be sufficiently small. In Calico, this means that there is a virtually unlimited amount of space, but the user must then content with a high amount of diffusion and low visibility. Intentional interfaces provides the means to manage the space needed to explore a design across multiple sketches to depict all parts of a design.

With Calico, there is the potential to address this issue using intentional interfaces by using several canvases to include all parts of a design. However, there is a further issue. Within the context of the informal design activities that Calico supports, designers typically only draw as much as they need to, which results in partial diagrams with minimal detail. As such, designers may create representations that have poor role expressiveness due to minimal detail. Very often, the original designer may need to be consulted in order to understand their sketch. However, content in sketches may be easier understood by looking at related content and hidden dependencies, which the intentional interfaces feature may help the designer do by navigating between canvases linked into chains. 


\subsection{Secondary Notation}

Secondary notation refers to the use of formalisms that deviate from primary established notations. Because of the freeform nature of sketching within Calico, users have the flexibility to overlay additional detail in sketches and use improvised notations as needed. Within Calico, users may choose to represent concepts using scraps with connectors as an alternative to basic sketching, in which case they can use the shape of scraps, as well as the colors of connectors to overlay meaning. Users may add annotations to scraps, such as drawing a symbol in the corner of a scrap, text content in the scrap with a color that has an assigned significance, or by using connectors in creative ways.


\subsection{Viscosity}

Viscosity refers to a medium's resistance to change. Similar to a regular whiteboard, the contents in Calico exhibit a low viscosity because content can always be erased and redrawn in any manner. 

Calico has three features which potentially reduce viscosity in comparison to plain sketching: content is moveable, gestures make manipulating content faster, and content can be copied. First, all sketched content can be moved using scraps, as opposed to the whiteboard where it must be redrawn. The capacity to move content means that sketched content is not ``locked in'' like a whiteboard, but instead content is less viscous because it can be adjusted. Second, with sufficient familiarity with scrap gestures in Calico, moving content can potentially be done quickly with minimal effort. A user may be less likely to modify content if accomplish that task requires a great deal of effort. In Calico, users do not need to switch modes or move their hand away from sketches, allowing them to move content fluidly without distracting them away from plain sketching. 


\subsection{Visibility}

Visibility refers to how easily all parts of a design are visible, or at least how easy it is to juxtapose two parts of a design \cite{Petre2013BookChapter}. As with a whiteboard, a design is highly visible so long as it is small enough to fit within a single space. In Calico, designs that are diffused across multiple canvases will have much less visibility than designs that are limited to a single canvas. The intentional interfaces feature attempts to address this issue by making it easier to navigate between canvses, and also to zoom out to the cluster view and view all canvases at once, albeit at a low-detailed perspective.

%Also, juxtaposing parts of a design is easier in Calico due to scraps, the palette, and intentional interfaces. Scraps allow users to reconfigure a diagram such that two parts can be placed next to one another by moving them or copying them. The palette allows parts of the design to be copied to other canvases to compare them. Intentional interfaces increases the ease of moving between parts of the design, and organizes the canvases so that it is easier to step through a design. Further, two canvases can be placed next to one another in the cluster view for comparison.

\subsection{Summary}

Overall, the major benefits of Calico over the whiteboard are that intentional interfaces allows more content to be preserved whereas on the whiteboard it would be erased, and the fluidity of manipulation offered by scraps. The benefits make content less viscous, more provisional, promote secondary notations, role expressiveness, and reduce error proneness. What is lost in these features is that designs may be more diffuse, and because of that diffuseness, require more hard mental operations, have less visibility, more hidden dependencies, and be harder to progressively evaluate. For each benefit gained through features, Calico loses the simplicity of the whiteboard, but attempts to mitigate these losses through the other features.

\section{Overall strengths and weaknesses}
\label{discussion:overall-strengths-weaknesses}

\DIFaddbegin \DIFadd{Having examined the features from a practical perspective in Section \ref{discussion:strengths-and-weaknesses} and at a theoretical perspective in Section \ref{discussion:cog-dim}, this section examines the overall strengths and weaknesses of Calico as a whole.
}

\subsection{\DIFadd{Strengths}}

\DIFadd{\textbf{Strength: It supported all design behaviors}
}

\DIFadd{All design behaviors were performed by users in the field at least once using Calico's features. While not all Calico groups performed every design behavior using the feature, for example only the research group used intentional interfaces to retreat to previous designs, they did use the features in their own work. Some features were used to perform design behaviors often, such as navigating between canvases using intentional interfaces, and creating low-detail detail diagrams using scraps. 
}

\DIFadd{Some features were used rarely, but served an important purpose in those moments. The OSS group only engaged in refining their sketches from plain sketches to UML diagrams once, but in that moment it allowed them to transition from a freeform sketch to a more formal one. The research group also brought their work together using scraps to tag what they did not and did not implement in source code, but using scraps enabled the tags to remain attached to the scraps as the diagram was modified. Collectively, the features were flexible enough to address all design features without requiring the users to shift their context or shift to different modes.
}

\DIFadd{\textbf{Strength: Supported them when they needed it as opposed to forcing it upon them.}
}

\DIFadd{While all features were used to support their respective design behaviors, the designers only used those features when it benefited them. As the second design behavior suggests, designers will only draw, or use formal notations, as much as they need, e.g., they will only write a name to represent a component rather than the entire details of that component. This behavior extends beyond simply what they sketch, and onto their use of Calico's advanced features: designers will not use the advanced features when they do not need them.
}

\DIFadd{Regardless how minimal the effort may be, each feature brings with it some cost that the user must pay in order to gain the benefit of that feature. Scraps enable rapid interaction with sketched content, but require the user to think in terms of objects and manually group them. Intentional interfaces enable the user to have more free space, but requires the user to think in terms of multiple canvases with low visibility. A user may not need the benefits offered by these features and may simply want to sketch without structure imposed on their work. A strength of Calico was that it allowed them to do that, but, when ready, to also convert regular sketches into scraps using the press-and-hold gusture, or partition canvases across several canvases by pressing the ``copy canvas'' button and gruop them using intentional interfaces.
}

%DIF > --> introduce quotes
%DIF > --> users could use modeling, but used free sketching. Many features were not used simply because they did not need them!
%DIF > --> refinement only happened once or twice, but when they used it, they needed it

\DIFadd{\textbf{Strength: Supports a wide range of design}
}

\DIFadd{All groups engaged in different forms of design, yet Calico supported all teams in performing their own work. For the OSS group, this meant designing new components and refactoring source code in a mature open source software system. In the case of the interaction designers, they were engaged in early phase requirements gathering by processing interviews which will be used to build personas for interaction design. In the third case, the research group, the design involved refactoring and designing the first version of a software tool in a globally distributed team. 
}

\DIFadd{These groups used Calico for their designs, but created their own culture in using the tool and created different diagrams. The OSS group created narratives explaining how data is moved through their system using multiple canvases. The interaction design group organized and tagged scraps, and copied canvases to generate multiple perspectives. The research group carried on their sessions remotely, and sketched dense diagrams using scraps. Calico supported the groups during the entire lifecycle of their sketches, which includes brainstorming, sharing, refining, and archiving }\citep{walny6069462}\DIFadd{. The groups adapted Calico to their existing workflow by sketching or importing diagrams, working, and later archiving their work by emailing or referring back to old sketches in Calico.
}


%DIF > --> three different teams in three different situations
%DIF > 
%DIF > --> It fits on top of existing design processes and tools

\DIFadd{\textbf{Strength: Can create a variety of diagrams}
}

\DIFadd{Across the design sessions, Calico showed that it could be used to create a variety of different types of diagrams. Some diagrams, such as box-and-arrow diagrams, benefit greatly from scraps because they could more easily be moved, related, and were space efficient. Other types of sketches, such as the image scraps placed along one and two dimensional plots benefit from being able to tag the elements with colors and move them along the plot to organize them. For complex diagrams, users can use plain sketching, but with scraps, they have a general purpose tool to group them and juxtapose them by moving them next to one another.
}


%DIF > --> they were juxtaposed
%DIF > --> they were different

\DIFadd{\textbf{Strength: Presents a minimal hurdle for first use}
}

\DIFadd{A strength in Calico's core design is that users can use its most basic features without training. New users unfamiliar with Calico may begin using it by dragging their finger across the board to create a stroke. The minimal overhead allows new users to begin working without the overhead of creating a new project, reserving a space, or investing themselves in extensive training. This is helpful when sharing or explaining a design to new users, who can begin participating in a design without any training. 
}

%DIF > Many of the feature are discoverable.
%DIF > people can walk up to it and use it

\DIFadd{\textbf{Strength: Interaction is fluid}
}

\DIFadd{Calico uses a minimal set of modes for manipulating content, which leads to more fluid experience than traditional digital sketching tools. Plain sketching is fluid because the designer is interrupted, at most, to reach for a new pen and can engage in a flow }\cite{csikszentmihalyi2009creativity} \DIFadd{with their content. The combination of scrap gestures for creating scraps and the bubble menu take a step towards the ideal standard of fluidity of the whiteboard by requiring fewer steps than traditional digital drawing tools to manipulating content. Feedback from the OSS group and the research indicated they prefered Calico's scrap interactions for manipulating content, but found it less precise than traditional drawing tools, while the interaction designers preferred traditional drawing tools for interaction.
}

\DIFadd{\textbf{Strength: The basic interaction mechanisms are strong}
}

\DIFadd{Overall, Calico contains a strong basic set of interactions. While Calico does not contain advanced drawing features such as shadows, blur, etc., nor does it have advanced modeling notations such as different types of arrows, shapes of boxes, etc., the groups did not find Calico lacking in basic features. Users valued the basic set of features in Calico, such as the eraser, undo/redo, colors, stroke thickness, and the set of Calico's advanced features. They found Calico's features sufficient to carry out their sessions, and found Calico's features natural to use after a period of time, in particular scrap gestures and the bubble menu. The same could not be said for the interaction scheme of Calico Version One, and users reported finding mode switching found in regular drawing tools awkward when used on the whiteboard.
}

\DIFadd{\textbf{Strength: It allows for creative uses that were not expected}
}

\DIFadd{A consequence of designing a flexible environment was that the groups used Calico in ways that were not envisioned in the original design of Calico. The use of Calico by a member of the OSS group to refactor their source code was a deviation from what was expected. Calico was envisioned to support the sketching and manipulation of abstractions, but the OSS group member found the writing on the electronic whtieboard good enough to write pseudo code and found the use of colors helpful as a secondary notation. The use of colors to tag scraps in both the interaction design group and research also came as a surprise. The use of Calico by the research group to keep track of what they had and had not implemented further corroborates Newell's notion that sketching serves as an external memory }\cite{Newell}\DIFadd{, but that specific use was not anticipated.
}

\DIFadd{\textbf{Strength: Free space is always available}
}

\DIFadd{Users could explore new ideas in Calico without having to erase old content in Calico. As mentioned in the previous cognitive dimensions session, this brings the advantage of allowing users to fully express their designs, but brings with it the problem of diffused diagrams that cause low visibility, hard mental operations, hidden dependencies, etc. While these negative consequences are concerns that should be addressed further, the value provided by easily available free space was an overall positive outcome for groups. During informal design sessions, the overhead incurred by needing to summarize designs within a single whiteboard may lead to ideas not being recorded. It is better that a designer need to search for a sketch they created, rather than not having the sketch at all.
}

%DIF > A limiting factor in many sessions is the amount of free space that designs have to sketch.
%DIF > 
%DIF > Of course, the more sketches that people create, the more diffuse their diagrams are.

\DIFadd{\textbf{Strength: Narratives can be created using canvases}
}

\DIFadd{The OSS group and one member from the research demonstrated that intentional interfaces is useful for creating narratives from sequences of canvases. Similar to how designers may refine sketches by converting lists into UML class diagrams, members from the two aforementioned groups manually reorganized the order of their canvases using canvas links in order to create a story from their sketches. These groups later shared their sketches, and compared the act of stepping through these canvases to giving a Powerpoint presentation.
}

\DIFadd{\textbf{Strength: Sketched content is always available}
}

\DIFadd{A quality of Calico that members from the OSS group and the research valued was always being able to access their canvases. Group members may not always have access to the physical equipment, and being able to access the content on their own computers was helpful. A member from the OSS group stated:
}

\begin{myindentpar}{1cm}
\DIFadd{\emph{``The fact that someone can work with their own tablet or computer, like [a member of the team] did with his alternative view, is something really powerful to do. If you try to do that in real life... we're blessed with two whiteboards, but some companies may only have one. It makes it harder to do that. Especially when someone is already at the whiteboard discussing something, and you want to bring in an alternative perspective but you need to wait until they're done doing whatever they're doing. That definitely makes groupwork easier.''}
}\end{myindentpar}

\DIFadd{The remote member of the research group further reported that being able to access the content from the weeklong intensive design session was helpful. 
}

%DIF > Important in the shared OSS setting. digital documents made content always available, but this made content freely available in its physical equivalent setting.

\DIFadd{\textbf{Strength: The system is distributed}
}

\DIFadd{Having synchronous collaboration be a fundamental feature was a strong point of Calico in the environments of the OSS and research groups. For the OSS group, this led to the scenario that members could sit around the electronic whiteboard and participate from a couch with a tablet in their lap. Members provided feedback in meetings informally by handing tablets back-and-forth and sketching over the diagrams, having their annotations displayed on the large electronic whiteboard. With multiple tablets, multiple team members could talk simultaneously without a single arbitor at the whiteboard blocking content, which is what traditionally occurs at the regular whiteboard }\cite{Shih2009}\DIFadd{. For the globally distributed research group, the consequence was that remote members participating using Skype could directly contribute by manually sketching their ideas, rather than remaining a passive contributor.
}

\DIFadd{\textbf{Strength: Users are not locked in to synchronous work}
}

\DIFadd{A person at the whiteboard that wants to write something, but cannot, may ``spin their wheels'' and ignore conversations until they can externalize their throught }\cite{Olsonb}\DIFadd{. A strength of Calico is that users can branch off into their own canvas, either by using creating a new canvas or copying the one they are actively using, and explore their own ideas independently. In meetings of large groups, this frees members to generate their own alternatives when inspiration strikes them, and later request everyone to join his canvas so that the designer may explain their alternative design.
}

%DIF > --> copy your work and do my own thing, then call you over
%DIF > --> synch and asynch, and how easily that's handled

\subsection{\DIFadd{Weaknesses}}

\DIFadd{\textbf{Weakness: Juxtaposition sketches on different canvases is difficult}
}

\DIFadd{A weakness in Calico is that sketches across canvases are difficult to juxtapose. Feedback from users indicated that they wished to refer to content from other canvases. In order to refer to past sketches, users either used the palette to copy sketches so that they are immediately available for reference, or they used the navigation buttons to rapidly jump between previous canvases and the canvas they are currently in. These other features were used to compensate for the inability to place sketches side-by-side.
}

%DIF > --> People want to put things side by side, but right now they can't.


\DIFadd{\textbf{Weakness: Users that do not invest in organizing canvases will experience difficulty in later locating their work}
}

\DIFadd{In previous work, there is much evidence that the Calico grid helps in design. It provided an intuitive interfaces to organize and partition work between canvases. We believe that  intentional interfaces is a step forward from the grid in the right direction for supporting design, however this feature has not had the opportunity to be polished and iterated on to improve its usability. One weakness in this iteration of intentional interfaces is that users that do not actively manage their canvases will likely have difficulty in knowing where their sketches will be located in the cluster view. Users that manually manage their canvases do so by linking their canvases using tags. Users that do not manage their canvases create radial clusters, with no name, that are difficult to identify. Future iterations of intentional interfaces in Calico will need to address this issue.
}

\DIFadd{\textbf{Weakness: Reviewing a large number of canvases is tedious}
}

\DIFadd{Moving across several canvases using the cluster view is a slow experience using Calico that can become tedious. A consequence of creating diffuse diagrams with many canvases is that the cluster view will need to adjust the size of canvases in order to display all canvases. The resulting effect is that images becoming too small to distinguish when there are a large number of canvases. Other features help to mitigate this problem, such as using the navigation buttons, and using the breadcrumb bar. However, members in the OSS group reported that, despite the slow interaction, they prefered to use the cluster view with previews to review sets of related canvases.
}

%DIF > In the current implementation, it makes images too small (intentional interfaces).

\DIFadd{\textbf{Weakness: Does not follow traditional interaction patterns, requires users to learn new gestures to manipulate objects}
}

\DIFadd{While the basic sketching functionality of Calico can be used by new users without training, the advanced feature set of Calico may take new users some time and practice to become proficient. A further issue is that users may approach Calico with a pre-conceived set of affordances that is based on their experience with other touch based devices. For example, members of the OSS group expected the large electronic whiteboard to use the same gestures as their mobile phone, while the interaction designers expected it to behave like Microsoft OneNote. Calico introduced another set of interactions that the users needed to learn. 
}

\DIFadd{\textbf{Weakness: It stops short of something that becomes an actual notation}
}

\DIFadd{While Calico supports users in creating approximations of formal notations such as UML, it stops short in delivering the behavior and expressive power of a formal modeling tool. A strength of Calico is that it supports working with informal notations and refining notations into more formal approximations by creating scraps and linking them with connectors. However, a designer may reach a point in which they are ready to add formal detail to their diagrams, such as double headed arrows, cardinality, and other formal relationships. While Calico does not currently provide support for the more formal elements, such support has been observed as being designed in late-stage designs, after a designer has committed to their design and begin recording lower-level, detailed design decisions.
}

%DIF > --> While we can do this and this, and seem like a UML diagram, the people will actually want UML functionality. We need to find a way to fold that in
%DIF > --> additional notational detail
%DIF > --> they want it near the end. It is a UML diagram, it is an interface diagram
%DIF > --> Up until I was doing something, then I'm doing a different type of design. While Calico is very good at making that transition from just sketching (words on paper) to actual diagram, it falls short in giving you the full notational support for that diagrm
%DIF > --> this issue plays out in a different sense. Meetings happen at different levels, in some meetings the transitions happen

\DIFadd{\textbf{Weakness: The setup is not lightweight}
}

\DIFadd{A problem observed in the setup of Calico was that, in order to use Calico, the system needed to be prepared and maintained by an administrator. While this is not specifically a weakness of Calico itself, it is highly relevant to Calico because, based on my observations, individuals will not use Calico if there is a delay in using the system. Instead, they will turn to using a normal whiteboard or a pad of paper. 
}

\DIFadd{In practice, the large electronic whiteboard needed a dedicated system that was always running so that each group could begin using Calico simply by turning the attached projector. In order to launch Calico on a personal tablet, the tablet required the installation of the Java Virtual Environment, which at times delayed meetings when new members needed to install their software. However, once group members performed the initial setup of Calico on their own machine, connecting to the Calico server became a more lightweight action.
}

%DIF >  what else do I see
%DIF >  feel free to remind people why this matters

\DIFaddend \section{Interviews with users}
\label{discussion:interviews}

%In the previous sections, I found that Calico's features adequately support the broad set of design behaviors and discussed the strengths and weaknesses of those behaviors. In this section, I corroborate those findings with a deeper discussion of the features. I do so by first performing a cognitive dimensions analysis of Calico's features at a hypothetical level. I then review the feedback from the interviews.

\DIFdelbegin \subsection{\DIFdel{Interviews}}
%DIFAUXCMD
\addtocounter{subsection}{-1}%DIFAUXCMD
\DIFdelend %DIF >  The interviews is mostly okay, just need to make it consistent
\DIFaddbegin 

\DIFaddend %//I've already made my interviews, now it's just my opinions
%It's clear that each group is using Calico very differently. They're finding different strangths and weaknesses. Interactions designers... not having an infinite canvas. When push came to shove, they made a lot of use of Calico. 
%There is the open source group, they didn't make big use of scraps, but they did represent code. Not our intention, but big use. One of the members used code because he had a large whiteboard.
%The research group. One of the guys did a lot of prep work.
%//the important part is that there is a lot of different uses of Calico, and Calico held up. All designs were created. 
%//Design group stated that not having an infinite canvas was a hurdle
%//what we got from the researchers were comments like this:
%//what we got fromt he oss group were like this:
%//if you pull out real quotes, it's stronger.
%//roll context into discussion. 
%
%Characterizing each session

In the previous section, I examine the capability of Calico's features to support design from the perspective of a cognitivie dimensions analysis. In this section, I discuss the feedback received on the features from the participants interviewed in each group.

From the interviews and observations, what is clear is that each group used Calico very differently. Each group used Calico to support different activities, with different amounts of people, and in different settings. These groups subsequently found different strengths and weaknesses in Calico.

The interaction designers used Calico to simulate activities that would have normally involved physical artifacts over a physical whiteboard. As such, many of the comments that came from the interaction design group related to Calico's strengths and weaknesses as they compared to the physical artifacts that the interaction designers normally used. For example, a strength that the interaction designers found was its ability to import images \DIFaddbegin \DIFadd{as image scraps }\DIFaddend and arrange them as they would have on a physical whiteboard. Explaining why this was important, one member of the interaction design group stated, 
\DIFdelbegin \DIFdel{``It's really hard to say 'that person or this person'. Much easier to speak to a face''. }\DIFdelend \DIFaddbegin 

\begin{myindentpar}{1cm}
\DIFadd{\emph{``It's really hard to say `that person or this person'. Much easier to speak to a face.''}
}\end{myindentpar}

 \DIFaddend They found that Calico's strengths were those that enabled actions not easily done with physical artifacts. For example, replication of artifacts encouraged the interactions designers to explore more perspectives than they would have\DIFaddbegin \DIFadd{, }\DIFaddend had they used actual physical artifacts. Calico's features further allowed them not only \DIFaddbegin \DIFadd{to }\DIFaddend arrange the images themselves, but also the sketches generated around those images, in which they could arrange and copy written text to organize the whiteboard. 

However, they found some experiences in Calico still not on par with using the physical whiteboard. They viewed the gestures of moving scraps too slow, where one member of the interaction group stated that 
\DIFdelbegin \DIFdel{``the overhead in manipulating was too much''. }\DIFdelend \DIFaddbegin 

\begin{myindentpar}{1cm}
\DIFadd{\emph{``the overhead in manipulating [scraps] was too much.''}
}\end{myindentpar}

 \DIFaddend Further, they found the space available in canvases being too small compared to the boards they normally use, stating that
\DIFdelbegin \DIFdel{they were ``blocked by the physical limitations of the }%DIFDELCMD < [%%%
\DIFdel{electronic}%DIFDELCMD < ] %%%
\DIFdel{board''. }\DIFdelend \DIFaddbegin 

\begin{myindentpar}{1cm}
\DIFadd{\emph{``[we] [were] blocked by the physical limitations of the [electronic] board.''}
}\end{myindentpar}

\DIFaddend Despite these obstacles, they overcame such obstacles, making dozens of canvases with sketches and scraps. They made real progress in a real design task within their professional work.

The OSS group took \DIFdelbegin \DIFdel{on a very }\DIFdelend \DIFaddbegin \DIFadd{a }\DIFaddend different approach to \DIFdelbegin \DIFdel{Calico, using }\DIFdelend \DIFaddbegin \DIFadd{using Calico, making use of }\DIFaddend it in group design discussions of three or more people, and also personal sessions\DIFdelbegin \DIFdel{over both software components and software code}\DIFdelend . A strength of Calico in this setting was to increase their capability to \DIFdelbegin \DIFdel{do more in the same }\DIFdelend \DIFaddbegin \DIFadd{share their meeting }\DIFaddend space. For example, intentional interfaces encouraged members to brainstorm more, where one member of the group states, 
\DIFdelbegin \DIFdel{``we're more willing to draw any random thing on there because we know that we can erase it, or go to a new canvas at any point. I would say more random ideas get thrown on there.'' }\DIFdelend \DIFaddbegin 

\begin{myindentpar}{1cm}
\DIFadd{\emph{``we're more willing to draw any random thing on there because we know that we can erase it, or go to a new canvas at any point. I would say more random ideas get thrown on there.''}
}\end{myindentpar}

\DIFaddend The OSS group in particular took advantage of free space to create dozens of canvases. For example, they consecutively chained canvases within intentional interfaces to generate sets of canvases that together formed narratives, such as how data steps through software components, or how a user interface behaves. 
\DIFaddbegin 

\DIFaddend Some of their use was also unexpected. The sessions involving source code came as a surprise, as I did not expect the developers to \DIFdelbegin \DIFdel{drop }\DIFdelend \DIFaddbegin \DIFadd{copy and }\DIFaddend paste screenshots of actual code into Calico, but these sessions became some of the most interesting, as they involved a great deal of impromptu notations to step through source code.

%``I had something where I needed to get my thoughts written out as far as I was trying to design...''
%``It was really useful to switch... I used a lot of color. It kind of helped me identify, like, which kind of objects are important. The other thing was that I wanted to look at the XML code that was relevant to this diagram, so I went on my machine, logged in over the web browser, took a screenshot of my code, and pasted it in to the canvas. That way when it came in over here on the board, I could look at the code from the screenshot, and do my work based on that. So that was very helpful.''
%``Having it in one page, one screen, was helpful.''
%``I ended up taking a different direction in the strategy I was working. So I just copied the canvas, I made an alternative, yeah, so that was nice. I made another screenshot of different code.''

The research group presented a third unique setting in which meetings were distributed, had a member that did a lot of design up front before the group design sessions began, and generated very large process diagrams that described the entire behavior of their software system. As opposed to the other groups, the research group members were more isolated from one another, and used meetings to review their work, \DIFaddbegin \DIFadd{and }\DIFaddend plan what to do next. The research group viewed one of Calico's strengths in their setting as being able to carry on meetings with remote members, where one member stated: 
\DIFdelbegin \DIFdel{``I think the biggest benefit was crossing space and time. Being able to pause and resume'', and ``the main benefit was that after I left, we were able to reference all those things that we created while I was there pretty easily..''. }\DIFdelend \DIFaddbegin 

\begin{myindentpar}{1cm}
\DIFadd{\emph{``I think the biggest benefit was crossing space and time. Being able to pause and resume...the main benefit was that after I left, we were able to reference all those things that we created while I was there pretty easily.''}
}\end{myindentpar}

\DIFaddend They viewed Calico as more beneficial than photos of a whiteboard, stating that 
\DIFdelbegin \DIFdel{``you can't go back and edit that.'' }\DIFdelend \DIFaddbegin 

\begin{myindentpar}{1cm}
\DIFadd{\emph{``you can't go back and edit that.''}
}\end{myindentpar}

\DIFaddend Further, one member of the research group prepared his work ahead of time for meetings, stating:
\DIFdelbegin \DIFdel{``I could have done this in a powerpoint as well''... ``it's much better }%DIFDELCMD < [%%%
\DIFdel{to have done it in Calico}%DIFDELCMD < ] %%%
\DIFdel{because later when I show it, I can have people changing it, and in powerpoint you cannot sketch and draw.'' }\DIFdelend \DIFaddbegin 

\begin{myindentpar}{1cm}
\DIFadd{\emph{``I could have done this in a powerpoint as well, [but] it's much better [to have done it in Calico] because later when I show it, I can have people changing it, and in powerpoint you cannot sketch and draw.''}
}\end{myindentpar}

\DIFaddend This particular member further found the visual structure of intentional interfaces helpful when reviewing work from past meetings, stating 
\DIFdelbegin \DIFdel{``if you're designing a complex thing with stages and you're trying to tell a story, you can say: okay we've tried that, would you like to see all this path we went through?'' }\DIFdelend \DIFaddbegin 

\begin{myindentpar}{1cm}
\DIFadd{\emph{``if you're designing a complex thing with stages and you're trying to tell a story, you can say: okay we've tried that, would you like to see all this path we went through?''}
}\end{myindentpar}

\DIFaddend Calico served as a virtual meeting space for the members that helped maintain continuity of meetings across the duration of the project.

%--- christian
%``we had one whole design that was thrown away''
%
%``designs get very complex... you want to keep a history of what you've done, the branches that you've pruned. Having a structure is essential. If you're designing a complex thing with stages and you're trying to tell a story, you can say: okay we've tried that, would you like to see all this path we went through? If you don't have the structure you'll have to create it somewhere else. If it's already here...''

The takeaway \DIFdelbegin \DIFdel{was that each session }\DIFdelend \DIFaddbegin \DIFadd{is that each setting }\DIFaddend introduced a unique way of using Calico. Each group had a different set of needs, a different setting and setup of people, and yet, Calico held up to a significant portion of their needs. Each group followed through with their design sessions and completed work in actual projects without abandoning their use of the tool. That is not to say that it was a perfect match for all groups. One of the teams, the interaction design group, did say that they would not continue to use Calico because its features did not match their needs, such as an infinite canvas and the system having difficulty with the number of images they used. This may either mean that their needs would be better suited by another tool, or further research is needed within the Calico environment.

Further, from the interviews, three observations stood out: \DIFaddbegin \DIFadd{(}\DIFaddend 1) Calico did not interfere with their normal design process \DIFdelbegin \DIFdel{, }\DIFdelend \DIFaddbegin \DIFadd{of the OSS and research groups, and despite difficulties, the interaction design group performed non-trivial design sessions, (}\DIFaddend 2) Calico deployments still have issues that need to be addressed that are intrusive, and \DIFaddbegin \DIFadd{(}\DIFaddend 3) that Calico led to positive changes in their design habits.

First, the OSS group and the research group reported that Calico and its features did not prevent them from carrying out their design as they normally would have, while the interaction design group reported that Calico somewhat got in the way of their normal design activity. The OSS group reported that they did not feel any loss of expressive control in using Calico in comparison to the whiteboard, and reported that they normally would have performed many of the same activities on the whiteboards available in the immediate area. The research group reported that they formerly carried out their designs on the whiteboard, and transitioned to using Calico for the same activities. The interaction design group reported that it closely approximated the whiteboard, but they found that the quality of sketch input forced them to write large and that the system lagged at times, which was distracting during their sessions.

%Did it prevent you from doing what you normally would have done?
%- quality of sketching on whiteboards was largest compromise. Tablets helped, but was not the same (1,2,3)

Second, there were some issues that were commonly intrusive among the three groups. Nearly all groups reported that the large electronic whiteboards diminished the quality of their own handwriting, forcing them to either write slower, or write larger. The OSS group connected to their Calico server using either tablets or their own computers, which they reported was preferred in producing clearer handwriting due to precise input. All groups used text-scraps to produce legible text as well. All groups also reported that launching Calico on their own machines was sometimes a hurdle. The OSS group reported that it was less of an obstacle for them because of the availability of tablets preloaded with Calico. The research group reported trouble setting up Calico on the laptops of new members of their group.
\DIFdelbegin \DIFdel{Lastly, the members of the interaction design group were accustomed to a different set of gestures and interaction design patterns from their own set of digital sketching tools, particularly OneNote, which led to confusion with Calico's features, such as how to use scraps and gestures.
}\DIFdelend 

%Features that detracted
%- going against the grain... people are accustomed to their existing tools.
%- too slow to bring up
%- turned to word documents to supplement activity

Third, all groups reported some positive changes to their existing design habits. All groups reported that they had a sensation of having more free space, and reported that they created more sketches than they normally would have as a result. The research group reported that they created more complex sketches as a result of scraps and connectors, which helped them address a deeper level of complexity in their design. Both the OSS group and the research group reported that, since they did not feel a need to delete unused sketches, they returned to old sketches more often.

\DIFdelbegin \section{\DIFdel{How well the features support the design behaviors}}
%DIFAUXCMD
\addtocounter{section}{-1}%DIFAUXCMD
%DIFDELCMD < \label{discussion:designbehaviors}
%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < simply about design behaviors, and how well calico's features supported it beyond regular whiteboard use.
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{Bringing together the reports of the design behaviors in Chapter \ref{chapter:evaluation}, Calico did indeed provide support for all fourteen design behaviors. As indicated by the Table \ref{chapter:evaluation:designbehaviors-table} in Section \ref{chapter:evaluation:design-behaviors}, not all features supported their intended design behavior, and indeed some features provided support for behaviors when they were not expected to, but, by and large, every single design behavior exhibited itself, and every single one was done using either scraps, the palette, international interfaces, or the fading highlighter. Not continuously by every single group, but the groups used the features when they could have simply sketched as they normally would have on the whiteboard. 
}%DIFDELCMD < 

%DIFDELCMD < %%%
\subsection{\DIFdel{Supporting the kinds of things they drew}}
%DIFAUXCMD
\addtocounter{subsection}{-1}%DIFAUXCMD
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{In most cases, scraps }%DIFDELCMD < \& %%%
\DIFdel{connectors, as well as the palette, supported the kinds of representations that the OSS group and the research group drew. Individuals from all groups used these features to create an variety of different representations (design behavior 1). The representations they created ranged from very low-detailed boxes with just names to complex representations made from scraps stacked on top of each other (design behavior 2) . While not happening often, some sketches did evolve into from simple hand written lists to complex structures involving box scraps and connectors (design behaviors 3). In many of these complex sketches, individuals created impromtu notations to concisely depict semantically rich concepts (design behavior 4). Together, both scraps and the palette were used along with regular sketching such that they provided the sensation of using a whiteboard, but with significantly lowered viscosity for changing them. In the rest of this section, I discuss the impact that supporting these design behaviors had on the design sessions, as well as cases in which support for the design behaviors did not work as well as they should. 
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{Pertaining to the first design behavior, scraps allowed several types of representations to grow ``organically''. Representations such as lists, software architecture representations, and user interface mockups received the most benefit from Calico's features. With respect to lists, scraps made handwritten representations more fluid to edit because content could be quickly moved. Temporary scraps made moving content a ``ready-at-hand'' action, which allowed users to make space to insert handwritten items in lists, move content out of the way, and copy repeated elements in lists without pausing their activities to switch modes. With respect to software architecture representations, scraps enabled users to more easily to grow diagrams in place. Users created complex box-and-arrow diagrams, and often moved scraps around the canvas to make space for more parts of the design. The scraps themselves were often very low detail, typically only containing the name of software component, but the relationships between the scraps were heavily discussed and edited. With respect to the user interface mockups, the scrap copy feature helped users more easily explore alternative user interfaces. Users could quickly create variations of a user interface. In comparison to a regular whiteboard, representations in Calico could be more easily iterated upon in-place in Calico, and in comparison to a formal tool, representations have much more flexibility in what they can represent.
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{In the case of the interaction designers, scraps fell short in supporting the the interaction designers working with the planar representations they created. These representations, however, were outside of Calico's original design scope. In planar sketches, a scrap's position on the horizontal and vertical axes define its relationship to other scraps. For example, the interaction designers drew a one-dimensional axis, and placed scraps according to their ranking on the axis from left-to-right. This is in contrast to box-and-arrow diagrams, in which the relationship between scraps is explicitly declared using arrows. There are numerous ways that scraps could be augmented to better support planar type representations, such as automated support for positioning scraps in one- and two-dimensional axes, tables, etc., but ultimately, the interaction designers were able to create these planar representations without Calico's features getting in their way. 
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{Pertaining to the second and fourth behavior, scraps }%DIFDELCMD < \& %%%
\DIFdel{connectors worked well to support users in conducting rich discussions over the runtime behavior of software components. Many of the conversations that took place were focused on how components pass data to one another, and these discussions were expressed using hand-drawn arrows, Calico connectors, and the fading highlighter. These discussions led to individuals improvising unique symbols in their representations to depict complex concepts. Further, I found it interesting that, while discussions were centered on the flow of data between software components, the users never recorded the paths taken by data onto the sketch itself, nor made any attempt to record it otherwise in their sketches. Formally, these paths could be captured using UML sequence diagrams, yet not once did I see this type of representation created. Based on my observations, I do not believe this was due to lack of foresight by the developers, but instead that sequence diagrams are too specific for the purpose of design at the whiteboard, and do not yield enough benefit to merit the time used to sketch them.
}%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < The research group demonstrated that Calico connectors could be used to successfully create dense process flow diagrams. Gestures made connectors fluid to create, and connectors that moved along with scraps made creating empty space on the canvas a natural action. 
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{Pertaining to the third design behavior, one of the scenarios in which Calico was the most useful was when individuals approached the board with a vague set of ideas, and used Calico to build new relationships and higher level concepts from that data over time. Much in the way that agile teams use Post-It notes to organize stories or qualitative researchers use affinity diagram to discover topics from data, members from groups used Calico to build relationships and categories from their work. At least once in each group, members turned to Calico with an unstructured set of data, and a goal of what they wanted to build from that data. After iterating over several canvases, they reached their goal by grouping and creating relationships between their data. 
%DIF < While not always used, the support by scraps and the palette for the first four design behaviors improved their capacity to build designs over their raw data.
%DIF <  One of Calico's strengths was to help teams process ``uncooked'' information.
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{The OSS group, for instance, found this activity valuable in their use of Calico. One member of the OSS group stated:
}%DIFDELCMD < 

%DIFDELCMD < \begin{myindentpar}{1cm}
%DIFDELCMD < %%%
\DIFdel{\emph{"We have different categories of stuff, and trying to figure out what relates to what. So we kind of just wrote it all out there, and just started almost randomly just drawing arrows to see what was what, we eventually deleted a bunch of connections, moved stuff around to make it cleaner looking."}
}%DIFDELCMD < \end{myindentpar}
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{The interaction design group encountered much the same situation within their experience. They did not use scraps to the degree that the OSS group did, but they used Calico's features in other ways to build relationships among their data. They imported images of dozens of individuals that they interviewed, and used this unstructured set of images to explore the patterns that appeared in their interviews. Where the OSS group used connectors to provide structure, the interaction design group instead visually organized faces into clusters according to their similarities in different categories. 
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{Pertaining to the fourth design behavior, while scraps supported many groups in improvising notations, users sometimes improvised box-and-arrow notations without the use of scraps. In particular, users sometimes chose not to use Calico connectors to show relationships between scraps. The OSS group, for example, did not use formal Calico connectors on many occasions, but rather sketched arrows by hand. While at first glance it may appear that the OSS members did not find Calico connectors sufficient to their needs, it was instead the case that, in those particular instances, the OSS group did not further need to evolve their software sketch. Thus they did not stand to benefit from making their hand-drawn arrows into Calico connectors, which enabled support such as remaining ``stickied'' to scraps as they are moved. In the instance in which the OSS group did evolve sketches involving components with arrows between them, they did indeed use formal Calico connectors. 
}%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < [+]It was a medium to process undigested information. The SW people did it in a meeting. Interaction designers did it. The onboarding researcher did it as well
%DIF < - "it ended out pretty different from where it started."
%DIF < - "it helped in visualizing things"
%DIF < - it didn't provide good support for lists, but at the same time people don't need much support for lists. scrap interaction provided good support for editing in place
%DIF < - Support for grouping content for the interaction designers... it did not support some of what they did, but they began to use some of the advanced functionality, such as linking content with formal connectors
%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < [-] It didn't quite support impromtu notations for software design, but it did for interaction designers.
%DIF < - palette did not quite support impromptu notations as I had hoped. Supported interaction designers, but software designers note as much. Software designers used the most creativity in establishing different kinds of connectors. A better feature would have been to include profiles, and define the meaning of profiles.
%DIF < - scraps lacked ability to be color coded, which a developed wanted to use to assign meaning
%DIF < - sw people use visual variables to assign meaning to their diagrams.
%DIFDELCMD < 

%DIFDELCMD < %%%
\subsection{\DIFdel{Supporting how they navigated their sketches}}
%DIFAUXCMD
\addtocounter{subsection}{-1}%DIFAUXCMD
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{While the interaction designers did not adopt Calico's features into their design designers pertaining to navigating their sketches, Calico's features did support the groups who did use the features, and those groups continued to use Calico's features across several design sessions. 
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{When working within canvases, scraps provided a supporting role in working with perspectives, alternatives, and abstractions. Scraps enabled users to examine different perspectives by moving and manipulating sketches, generate alternatives by creating multiple copies, and also were copied across canvases that represented different levels of abstraction of one another. The fading highlighter also excelled in supporting mental simulations over designs and explaining designers, two design behaviors which were quite often done together.
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{When working across multiple canvases, intentional interfaces played a strong role in helping people move between perspectives, alternatives, abstractions, reviewing their progress, and retreating to past designs. Intentional interfaces made it easier for users to jump to a new space to continue designing, and provided a structure to the design space as they moved between the parts of their design. The palette did not quite support these particular design behaviors as it was intended, but rather the palette played a role in helping users juxtaposing content across canvases, where intentional interfaces itself did not sufficiently support users. Intentional interface's inability to support juxtaposition also likely led to it only weakly supporting the design behavior involving bringing work together, but intentional interfaces did adequately support switching between synchronous and asynchronous work. 
}%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < With the exception of the interaction designers, the intentional interfaces feature sufficiently supported users in navigating between their sketches. 
%DIF < For those that used it, intentional interfaces and scraps both provided support for the fifth, sixth, and seventh design behaviors, which involved moving between perspectives, alternatives, and levels of abstraction.
%DIF < - intentional interfaces worked, scraps did too for a different reason
%DIF < - palette not so much
%DIF < - Again, they did use it to navigate between canvases. The OSS group, half of the research group did use it and found it use full. Other half of research group, and interaction designers did not.
%DIF < - <<these were all trying to address the same problem>>
%DIF < - "this was nice, being able to copy this scrap, it allowed us to show different use cases, right."
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{A notable contribution of intentional interfaces was that it provided an ordering to the canvases by linking and tagging them, which had a positive impact on the design sessions. This quality helped members from multiple groups to manage their different perspectives, distinguish alternatives, and step through multiple levels of abstraction. Users did not always remember where exact canvases were located, but they had a sense of where they were located within chains. Establishing an order between their canvases enabled them to frame their designs in terms of a story, in which the canvas closest to the center of the ring presented the initial exploration, and the canvases near the end of the chains contained content that contained final design decisions or summarized content from the previous canvases. Users mentally thought of moving between cells in the chain as moving ``backwards'' and ``forwards'', which, while it helped them form a story, resulted in confusion with the buttons to navigate through the history of visited canvases.
%DIF < - So much so, that it changed people's expectations of features, such as the backwards and forwards buttons.
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{Where the ordering of intentional interfaces fell short was when the canvases did not portray a linear story. For example, a single canvas may have multiple related canvases in which one was an alternative and another was different level of abstraction. Linear canvases did not capture the relationship between these canvases as well they could have. An alternative representation to the linear list may be to represent the list as a hierarchical tree, but this may not adequately capture the full relationship among the canvases as the canvases relate to one another in multiple and cross cutting ways. Any method used to add structure to the canvases should be lightweight to allow these different categories to emerge.
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{It was interesting to note that intentional interfaces encountered issues with scaling when chaining was not used. One cluster belonging to the research group grew to 34 canvases, but did not use clusters. Subsequently, all members of the research group experienced difficulty finding canvases within this cluster because they could not remember where they were, and the canvases in the radial layout were too small to recognize. In contrast, members of the research group had other clusters with just as many canvases, but could more easily locate content because content within the cluster was grouped into chains. The chains were visually distinguishable at a glance, and members could recall the design sessions in which they created the canvases within particular chains.
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{While the features met their criteria for support in two of the groups, they fell short in supporting the interaction designers in supporting several of the design behaviors pertaining to navigating between sketches. This may have been due to several situational factors. First, the sketches produced in this particular activity of interaction design may not lead to several of the design behaviors the features were intended to support. Design behaviors such as generating alternatives and navigating between different levels of abstraction are behaviors that have been observed during software system design. The interaction designers were engaged in a different type of design, which was to build a set of personas. This type of design involved generating several perspectives that are independent of one another, whereas a change in one perspective in a software system directly effect other representations of the same perspective. Second, the design sessions of the interaction designers only involved two individuals who shared a single electronic whiteboard. Features such as the fading highlighter, and design behaviors, such as those pertaining to collaboration, may only occur when several individuals participant in a design session, and members are using several devices. Third, the interaction designers already have a strong culture around how they use electronic sketching tools, which has a different mechanism for navigating between content. They use OneNote, which allows for scrolling between documents. This may have conflicted with Calico's method of navigation, as well as their comment that they wanted all of their sketches visible at once. However, despite these problems, the interaction designers were able to create nearly two dozen canvases with different perspectives.
}%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < A few factors contributed to the features not supporting navgiation as well as they could have. Interaction designers were designing in a different domain, leading to different needs. Juxtaposing sketches was difficult, which led to other ixxues. Some features require specific conditions to be necessary. sometimes you need to plan ahead
%DIF < Interaction designers didn't use them because...
%DIF < - not as complex domain as software
%DIF < - also different culture
%DIF < - Wanting all parts visible...
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{A further factor was that intentional interfaces became helpful in longer term projects. The interaction designers did not engage Calico for long term projects, having only used it intensely over the course of a week, and later for a very short period. In contrast, the OSS group used it for several weeks, and the research group for several months. One of the OSS group members commented on the importance of preserving his work for later use, stating that, ``I pretty much always, when I was done, went back to my computer and exported it as a PNG, and saved it so I had a record". The research group further retreated to their past designs after several months had passed, citing that they wanted to refer to decisions that they rejected in the past in order to not repeat the same mistakes in future iterations. 
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{It is interesting to note that content in Calico was used for a short period of time, but remained on the server for long periods of time. This is in contrast to the whiteboard, which has been shown to have short-lived, ephemeral content }%DIFDELCMD < \cite{cherubini2007let}%%%
\DIFdel{. Further, research that examined settings in which whiteboard content could be automatically saved and reloaded later led to individuals enforcing higher quality control on their own content in anticipation of using it later }%DIFDELCMD < \cite{Branham2010fromwhiteboard}%%%
\DIFdel{. I did not observe this to be true of sketches in Calico, instead these sketches remained the same quality and members of the OSS group and research group did not change their behavior in anticipation of reusing content. It is possible that this difference came about because users of whiteboard archiving systems require the user still need to erase their whiteboard and can only use images. In Calico, they can go back and make additions, which mentioned as useful by all groups (including the interaction design group, even though they did not return to content).
}%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < Require specific settings.
%DIF < [+] Retreating to ideas happens in very long term projects.
%DIF < - highlighter is useful with teams and multiple devices
%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < Bringing ideas together in Calico is not an easy action.
%DIF < - very hard to juxtapose items across canvases
%DIF < - juxtaposing an important part of bringing together, not as well done
%DIF < [-] Scraps and the palette helped juxtapose, but intentional interfaces did not address this design behavior as intented. This also likely affected bringing work together.
%DIFDELCMD < 

%DIFDELCMD < %%%
\subsection{\DIFdel{Supporting collaboration}}
%DIFAUXCMD
\addtocounter{subsection}{-1}%DIFAUXCMD
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{On the whole, the intentional interfaces feature served to support the design behaviors pertaining to supporting collaboration. Intentional interfaces supported individuals in moving between synchronous and asynchronous work. The fading highlighter supported extensive discussions and explanations of designs in group settings. For the final design behavior, intentional interfaces did not support bringing together work as well as it should have, however, scraps surprising provided a supportive role in this behavior. Also, the interaction designers did not use any of Calico's features in support of collaboration. This factor was influenced by a work culture of designing in pairs of two individuals, which is too few people to cause collaborative design behaviors, and a strong existing culture for collaborating on content on the whiteboard.
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{In the cases where intentional interfaces was used, it served to coordinate the workspace, allowing individuals to coordinate working together synchronously, or deviating to a separate canvas to work asynchronously. As stated in the results from Calico Version One in Chapter \ref{chapter:calico-version-one}, users that cannot access the board while someone else is speaking will ``spin their wheels'' and try to hold on to a thought until write their own idea. Intentional interfaces directly addressed this concern, allowing individuals to directly write on their own tablet or jump to their own space by pressing ``new canvas''. The OSS group, who shared a physical space at their office, had the most to gain from this functionality because they could access their sketched in meetings that they generated while on their own. One member of the group stated:
}%DIFDELCMD < 

%DIFDELCMD < \begin{myindentpar}{1cm}
%DIFDELCMD < %%%
\DIFdel{\emph{``the fact that someone can work with their own tablet or computer, like [a member of the team] did with his alternative view, is something really powerful to do. If you try to do that in real life... we're blessed with two whiteboards, but some companies may only have one. It makes it harder to do that. Especially when someone is already at the whiteboard discussing something, and you want to bring in an alternative perspective but you need to wait until they're done doing whatever they're doing. That definitely makes groupwork easier.''}
}%DIFDELCMD < \end{myindentpar}
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{This design behavior directly led to individuals further needing to explain the designs they generated on their own, or spur individuals in meetings to ask others about content they discovered by looking at new canvases they found in the cluster-view. While both the OSS and research group usually gestured with their hands, the fading highlighter was used a fair amount to walk through diagrams. However, it was most often used to explain content that was prepared beforehand, in which the users switched to using the fading highlighter, and did not switch to drawing mode during their explanations. 
}%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{Further, the quality of drawing content was important in the use of the fading highlighter in explanations. Users drew arrows at components, circled names of software entities, and wrote words using the fading highlighter. Users from both the OSS group and the research group valued the ability to temporarily draw, as opposed to using a bright-red dot pointer that does not leave a trail, which is the method used by commercial presentation tools such as Microsoft Powerpoint }%DIFDELCMD < \cite{WinPowerPoint}%%%
\DIFdel{.
}%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < Supporting synch and async
%DIF < - [+] It supports long term collaboration. People used it at both their desks and at the board.
%DIF < "being able to work simultaneously is nice."
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{While the other behaviors were well supported, bringing work together was likely the least supported design behavior by Calico's features was the fourteenth, bringing work together. The expectation was that users would copy and merge content from previous canvases. Instead, users merged ideas verbally and used those conversations to sketch new content. For example, OSS group did so by navigating to previous cells, then drew several use cases on new canvases to consolidate their previous discussions. The research group surprisingly used scraps to bring together work that was done outside of Calico as well. One research group member described using this during their week-long intensive design session: 
}%DIFDELCMD < 

%DIFDELCMD < \begin{myindentpar}{1cm}
%DIFDELCMD < %%%
\DIFdel{\emph{``...the rest of the week we'd basically be coding along trying to implement all this, sort of diffing what's in the code against this [gesturing to calico]. So we'd glancing back at this trying to understand... sort of like `oh, here's the new pieces that we added and here's the pieces that we will be adding, and refering this, here's the bit that we haven't yet implemented'''}
}%DIFDELCMD < \end{myindentpar}
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdel{It is possible that the intentional interfaces feature did not adequately support bringing together work because of its lack of fluidity in juxtaposing sketches. Users in the OSS group and research group turned to the palette for juxtaposing elements sketches across canvases, but such an action takes several steps to copy the contents to be juxtaposed to another canvas. Alternatively, users also used the back and forward button to navigate to recently visited canvases in order to compare the current canvas with previous canvases, but such a solution puts the burden on the user to remember previous canvases. Instead, the users should have been able to place canvases side-by-side in the cluster view, which would have allowed them to bring work together from multiple canvases simultaneously.
}%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < Did it encourage you to do anything that you wouldn't have done?
%DIF < - return to sketches...
%DIF < - performed sketches as they normally would have, but was able to make more complex versions (3)
%DIF < - all reported that they created more content than they would have on the whiteboard (1,2,3)
%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < Overall, with exceptions because of hardware limitations and performance issues, all groups reported that the features did not intrude on their normal design activity, and that it brought positive benefits. 
%DIF < 
%DIF < //do something with this later
%DIF < Calico played a complementary role to existing tools and practices for each team, serving as an external thinking space for individual and collaborative design. In each case of each team, the members turned to Calico to work on ``wicked problems'' in which the solution was not readily apparent, and required some exploration of the design space before arriving at the solution. Sometimes these problems involve trying to make sense of raw data before a software system is built, as the interaction designers did. Sometimes it is to make modifications to an existing system, as OSS group developers did. Other times they may be building a new system from scratch, as with the research group. Given that the studies each provided glimpses into different pieces of the lifecycle for each group's project, it is difficult to draw a direct comparison of the differences between the approaches of each group. However, several commonolities arose in Calico's role throughout its use for each team. Calico use was not restricted to any single phase of design, but was flexible enough to support groups at different points in their design process.
%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < \section{Minimally Invasive}
%DIF < \label{chapter:discussion:minimally-invasive}
%DIF < 
%DIF < In order to a determine whether Calico's features were minimally intrusive, or at least the degree to which they were considered so, I both report on responses from the interviews, and perform a cognitive dimensional analysis of Calico in use. In the interviews, I asked all groups if Calico interfered with their existing design habits, if they found the features intrusive, and if using Calico led to any positive changes in the way they design. In the cognitive dimensions analysis, I perform a detailed review of Calico Version Two with respect to each dimension. I consider Calico's potential to support each dimension in comparison to the whiteboard, and review how each dimension manifested within the observed design sessions.
%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < \subsection{Interviews}
%DIF < 
%DIF < % Refere to marian's book chapter
%DIF < In this section, I review the cohesion of the features together, i.e., how well each feature supports one another. I review the findings in Chapter \ref{chapter:evaluation}, reporting on what combinations of features that groups found helpful. For each feature, I review the support they received from other features. 
%DIF < 
%DIF < The results are summarized in Table \ref{table:cohesiveness}. The table can be interpreted as follows, the row header represents the feature under review, the column header represents the feature which supports it, and the intersecting cell represents the groups which found that the feature in the column header did indeed support the feature in the row header. Within the table, ``OSS'' represents the OSS group, ``IxD'' represents the interaction design group, and ``Res'' represents the research group. As an example, for the row header ``Basic sketching'' and the column header ``Scraps'', both the OSS group and the research group found that basic sketching was improved by the scraps feature. 
%DIF < 
%DIF < Each feature was supported as follows:
%DIF < 
%DIF < \begin{table}
%DIF < \centering
%DIF < \caption{Cohesiveness of features: Degree to which features supported one other.}
%DIF < %\begin{tabular}{|c|c|c|c|c|c|}
%DIF < %\begin{tabular}{ |p{2cm}|p{2cm}|p{2cm}|p{2cm}|p{2cm}|p{2cm}|}
%DIF < \begin{tabular}{ |p{2cm}|p{2cm}|p{2cm}|p{2cm}|p{2cm}|p{2cm}|}
%DIF < \hline
%DIF < &\multicolumn{5}{c|}{\textit{supported by feature within group}} \\
%DIF < \hline
%DIF < &Basic sketching &Scraps &Palette &Intentional interfaces & Fading highlighter	\\[5ex]
%DIF < \hline
%DIF < Basic sketching &  &	OSS, Res &OSS, IxD, Res &OSS, Res & OSS, Res	\\[5ex]
%DIF < \hline
%DIF < Scraps & OSS, Res &  & OSS, IxD, Res &  & OSS, Res	\\[5ex]
%DIF < \hline
%DIF < Palette & OSS, IxD, Res & OSS, IxD, Res &  &  &	\\[5ex]
%DIF < \hline
%DIF < Intentional interfaces&  &  & OSS, IxD, Res &	 & OSS, Res	\\[5ex]
%DIF < \hline
%DIF < Fading highlighter& 	 &	 &	 &	 & \\[5ex]
%DIF < \hline
%DIF < \end{tabular}
%DIF < \label{table:cohesiveness}
%DIF < \end{table}
%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < \section{Other findings}
%DIF < \label{chapter:discussion:other-findings}
%DIF < 
%DIF < //Section removed for the time being.
%DIF < \subsection{Context of Calico within the software design process}
%DIF < 
%DIF < Calico played a complementary role to existing tools and practices for each team, serving as an external thinking space for individual and collaborative design. In each case of each team, the members turned to Calico to work on ``wicked problems'' in which the solution was not readily apparent, and required some exploration of the design space before arriving at the solution. Sometimes these problems involve trying to make sense of raw data before a software system is built, as the interaction designers did. Sometimes it is to make modifications to an existing system, as OSS group developers did. Other times they may be building a new system from scratch, as with the research group. Given that the studies each provided glimpses into different pieces of the lifecycle for each group's project, it is difficult to draw a direct comparison of the differences between the approaches of each group. However, several commonolities arose in Calico's role throughout its use for each team. Calico use was not restricted to any single phase of design, but was flexible enough to support groups at different points in their design process.
%DIF < 
%DIF < Calico proved to be a helpful tool for mentally processing ``uncooked'' ideas and information. Individuals used Calico to investigate concepts, discover relationship between data, and also to provide structure over the data. Members from both the OSS group and the research group began some of their design sessions by listing entities, and refining those entities into box-and-arrow diagrams. They arrived at mockups of user interfaces and software structures by performing some or all of the indentified design behaviors, such as refining their sketches, navigating between multiple perspectives of sketches, and explaining their designs to one another. The interaction designers began with pictures of faces of the fifty people they interviewed, and developed categorizations over interacting with those images to produce personas. Both the OSS group and researchers also used Calico to iterate on existing solutions by importing screenshots of source code and screenshots of their user interface.
%DIF < 
%DIF < While Calico was considered a provisional environment, it served as an informal stepping stone to enacting change. Design decisions and plans of action were proposed while using Calico in meetings, but were not solidified until those decisions were recored into a formal document, or being implemented by a developer. By remaining provisional, developers and designers could use Calico as a ``playground'' to explore ideas without commitment, and do so more quickly without needing to worry about precision or completeness. The impromptu, lightweight, and flexible nature of freeform sketching, and by extension Calico, make it a medium which is easier to explore concepts. 
%DIF < 
%DIF < From these assertions, there are several benefits that Calico could provide within the software design process. First, in order to support the processing of ``uncooked'' ideas, Calico could improve its ability to import data. Importing images has already shown to be useful. This ability could be pushed further by pointing Calico directly at other sources of information, such as source code repositories. Second, given that design decisions are proposed within Calico and recorded elsewhere, there is an opportunity to provide a tracibility back to Calico. While the reserach presented showed that sketches will not remain consistent with the most recent decisions because they need to be manually updated, it in fact may be a desirable quality because it reveals past history. Providing traceability to old designs in Calico may reveal design history that would otherwise not be available. Lastly, retaining the history of sketches may be helpful to help designers avoid repeating the same mistakes.
%DIF < 
%DIF < --> Something about physical objects here.
%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < //Section removed for the time being.
%DIF < \subsection{The role of multiple devices}
%DIF < 
%DIF < An important enabling aspect of Calico was that it allowed everyone in a group to sketch on the whiteboard at the same time. The electronic whiteboards had the limitation of only allowing one person to write at a time, which was reported as inhibiting in Chapter \ref{chapter:calico-version-one}, but other users could contribute from their own laptop. This was addition, as past research has shown that groups of individuals will contribute significantly more ideas to a discussion than if only one person can write at a time [Shih 2010].
%DIF < 
%DIF < Individuals across all groups participating in a shared session with their own device reported a greater ability to participate in sessions in comparison to their previous meetings. In some cases, such as the interaction design group, team members continued to not edit content themselves. The interaction designer group previously reported that in a typical case, one member draws on the whiteboard and other provides verbal feedback. While with Calico this interaction remained largely the same, in select cases the member that provided verbal feedback participated from their laptop by copy and pasting his notes into Calico using text-scraps. The OSS group and the research group, however, reported larger benefits. The OSS group reported that having several tablets liberated people from needing to focus on the same content as the speaker, and could diverge into their own sketches, reference past sketches, and participate in sketching without reaching over the should of the person at the large electronic whiteboard.
%DIF < 
%DIF < Calico enabled individuals share the same space more easily. In the case of the OSS group, team members were able to share the same space to carry on their design sessions concurrently using different devices. Members reported that they moved between using Calico on the large electronic whiteboard, their computer, and the tablets. For example, two individuals the large electronic whiteboard, while another developer concurrently used a tablet at his desk. The research group also shared the same meeting space to conduct multiple projects using Calico.
%DIF < 
%DIF < Individuals also prepared content in Calico ahead of time on their own computers. Members from the OSS group and research group copy and pasted content into Calico, and later moved to the large electronic whiteboard to sketch over their imported artifacts. Artifacts ranged from source code, screenshots of user interfaces, and powerpoint slides. 
%DIFDELCMD < 

%DIFDELCMD < %%%
%DIF < %//This section is removed for the time being
%DIF < \subsection{Preserving context}
%DIF < 
%DIF < A general fallacy of informal tools, such as whiteboards, is that the context of the artifacts produced is not preserved. The transient nature of the whiteboard results in previous iterations becoming lost as they are manipulated in place or erased. Much of the rationale behind the final iteration is often not recorded in the sketch itself, but instead resides only in the heads of the designers present in the meeting. The sketches themselves become cryptic to anyone who did not create the sketch, and eventually become cryptic to even the original sketcher if enough time elapses to forget.
%DIF < 
%DIF < Members from the OSS group, interaction design group, and research group all remarked that Calico, and to some extent, intentional interfaces, helped them recover past rationale by illuminating details from their past sessions. They deleted fewer canvases, which provided them with more sketches to reference. Rather than creating new iterations in-place and overwriting past iterations, they iterated on their design on newer canvases, establishing a history that would not have been saved on whiteboards. With more canvases, members from all groups had more sketches available to remind them of design paths explored. In the case of the OSS group and the research group, they created chains of canvases that provided further context. The chains served to group canvases into design sessions, and also provide an ordering to the canvases, which either took on the meaning of reminding the sketchers of the chronology of the canvases within the design exploration, or allowed the canvases to build on the content of the previous canvases. Members from the OSS group and research group both used these ordering, and tags, to provide cues to the meaning of content within the canvases. Most canvases were tagged as a ``continuation'', but they found the ``alternative'' tag helpful in explicitly marking a canvas as such. They also remarked that naming canvases was important in remembering their purpose.
%DIF < 
%DIF < Despite the affordances that Calico and the intentional interfaces did provide, users remarked that there were several moments in which they wanted to declare more details, but were unable to do so. When linking canvases by tagging, members of the OSS group and research group wanted to provide direct links between canvases. For example, they wanted to label three canvases as being an alternative of the same canvas. Or, they desired to use multiple tags on a canvas. Further, design was not always linear, but rather a canvas may have references that lead to several other canvases. For example, the OSS group depicted a set of components, and defined the detail of those components in later canvases. Other group members desired to insert notes and annotations within canvases, but did not do so because there was not enough space.
%DIF < 
%DIF < Some of the context of design sessions could not be captured due to the limitations of the technology. Reviews confirmed that much of the design that takes place with Calico happens by talking over designs, pointing at figures, and gesturing to nothing in particular. One remote collaborator reported that being able to see participants using video conferencing tools was important because it allowed him to see the gestures of others. In the OSS group, usage logs showed that users viewed canvases without interacting with it for long periods of time, and interviews confirmed that developers were using the image to discuss the next steps. The sketch played an important role, but no evidence of this was evident from Calico without asking the developers. In a similar situation, usage logs demonstrated a intensive periods of usage for the highlighter in the OSS group, but upon playing back a slideshow of the usage, members of the OSS group could not recall the topic they used it to discuss.
%DIF < 
%DIF < In addition to the information available in Calico, the usage logs contained information that may have provider deeper context about sketches in future versions of Calico. The usage log provided intermediate screenshots of sketches, which revealed unique representations that provided information about how they led up to a final design. The logs further provided summaries of sessions, reporting when sketches were created, and the amount of time spent in creating the sketch. 
%DIFDELCMD < 

%DIFDELCMD < %%%
\DIFdelend \section{Summary}
\label{discussion:summary}

Having presented my analysis of the experiences reported in Chapter \ref{chapter:evaluation}, I now return back to the research question in Chapter \ref{chapter:research-question}. The goal of Calico was to create a minimally invasive, small set of features that worked together to address the full set of design behaviors. Rather than create a new feature for each design behavior, I created a small set of four features that address all fourteen. What I have is a small set of features which work together to address all design behaviors.

Addressing the first part of the research question, the observations made thus far show that Calico's features were indeed minimally invasive in real world  design activities. Minor issues did exist that users found invasive, for example limited space in a canvas, poor quality of handwriting on the large electronic whiteboards, and slow movement speed of image scraps for the interaction designers. Despite these issues, all users were able to use Calico to perform actual work in real design sessions, and users saw Calico as improving their capability to design in comparison to the regular dry-erase whiteboard.

Addressing the second part of the research question, most of the features did coherently work together. Table \ref{table:discussion:minimally-invasive} succinctly summarizes the contributions of each feature to the set of fourteen design behaviors. Each feature supports multiple design behaviors, and they clearly are integrated, and few of the features stand alone. Scraps and the palette are very much integrated, where both work together in design behaviors 2, 4, 6, and 14. Scraps were also integrated with intentional interfaces, both supporting design behaviors 5, 6, 7, and 9. The fading highlighter, however, stands apart from the other features, supporting design behaviors 8 and 13. While it stands alone, it is powerful in what it does, and does so in a minimal and lightweight fashion.

\begin{center}
\begin{longtable}{|p{4cm}|p{4cm}|p{8cm}|}
\caption{The set of design behaviors and the features that support them}\\
\hline
\textbf{Design Behavior} & \textbf{Supporting Feature} & \textbf{Design Principles} \\
\hline
\endfirsthead
\multicolumn{3}{c}%
{\tablename\ \thetable\ -- \textit{Continued from previous page}} \\
\hline
\textbf{Behavior} & \textbf{Supporting Feature} & \textbf{Design Principles}\\
\hline
\endhead
\hline \multicolumn{3}{r}{\textit{Continued on next page}} \\
\endfoot
\hline
\endlastfoot
1. They draw different kinds of diagrams	&Scraps \& connectors	& + Supported lists, box-and-arrow, UI diagrams well\\
\hline
\multirow{2}{4cm}{2. They produce sketches that draw what they need, and no more}&Scraps \& connectors	& + Represented software components as scrap and connectors, very little detail included. \\
\cline{2-3}
&Palette 	& + Reused low detail scraps they created across canvases\\
\cline{1-3}
3. They refine and evolve their sketches over time	&Scraps and connectors& + Plane sketches could be made into scraps, have detail added to them, and add connectors\\
\cline{1-3}
4. They use impromptu notations	&Scraps \& connectors	& + Color took on meaning in connectors

+ Scraps were tagged/underlined with color\\
\cline{2-3}
	&Palette	& + Visual icons were created using scraps and used repeatedly\\
\hline
%	&Scraps and connectors &Elementary visual looks and behavior can be randomly composed	&\\
%\hline
\multirow{2}{4cm}{5. They move from one perspective to another}&Scraps \& connectors	& + Different notational conventions can be mixed and matched on a single canvas
\\
\cline{2-3}
	&Intentional Interfaces	& + Users can explicitly request a new canvas to work on a perspective

+ Canvases are linked and ordered, helping find related perspectives	
	\\
\hline
6.      They move from one alternative to another	&Scraps \& connectors	& + Different alternatives can be quickly constructed by copying and moving and otherwise manipulating scraps and connectors\\
\cline{2-3}
	&Palette	& + Different alternatives can be quickly constructed by reusing elements from the palette and composing them differently \\
\cline{2-3}
	&Intentional Interfaces	& + Users can explicitly request a new canvas to work on a different alternative

+ Canvases are linked and ordered, helping find related alternatives	\\
\hline
7.      They move from one level of abstraction to another	&Scraps \& connectors	& + Different abstractions can be quickly constructed by copying and moving and otherwise manipulating scraps and connectors	\\
\cline{2-3}
	&Intentional Interfaces	& + Users can explicitly request a new canvas to work on a deeper level of abstraction

+ Canvases are linked and ordered, helping find related canvases of different levels of abstraction	\\
\cline{1-3}
8.      They perform mental simulations	& Fading highlighter	& + Users can use the highlighter to mark up their diagrams without editing them	\\
\hline
9.      They juxtapose sketches	&Scraps	& + Uses can move perspectives, alternatives, and abstractions next to one another by moving scraps\\
\cline{1-3}
10.  They review their progress	&Intentional Interfaces	& + Users can step back and examine their progress and process, overall and in parts, in the intention view\\
\hline
11.  They retreat to previous ideas	&Intentional Interfaces	& + Users can choose to enter one canvas in the intentional view or make a new canvas at any time\\
\hline
12.  They switch between synchronous and asynchronous work	&Intentional Interfaces	& + Users can choose to enter one canvas in the intention view, or make a new canvas and work separately\\
\hline
13.  They explain their sketches to each other	& Fading highlighter	& + Users can use the highlighter to draw attention to certain parts of a canvas	\\
\hline
14.  They bring their work together	&Scraps \& connectors	& + Scraps can pick up sketches and drop those sketches onto other scraps, merging them\\
\cline{2-3}
	&Palette	& + Designers can place sketches from different canvases into palette and later merge them into a single canvas
\label{table:discussion:minimally-invasive}
\end{longtable}
\end{center}

%Do these features fit together, and do I believe that they fit together
Addressing the third part of the research question, the set of four features did indeed support all design behaviors. Scraps and the palette supported designers in the kinds of sketches they created (design behaviors 1 -- 4). They allowed diagrams to ``grow organically'', have rich coversations over the runtime behavior of components, and build relationships over unrelated sets of data. Intentional interfaces, scraps, the palette, and the fading highlighter helped support the way designers navigated between sketches (design behaviors 5 -- 11). Intentional interfaces helped designers find relationed perspectives, alternatives, and levels of abstraction, and did so by attaching related canvases with tagged links and providing an order to them, which also helped to review their progress and retreat to previous ideas. The fading highlighter further supported designers in mentally simulating over box-and-arrow diagrams. Lastly, Intentional interfaces, the fading \DIFdelbegin \DIFdel{hihglighter}\DIFdelend \DIFaddbegin \DIFadd{highlighter}\DIFaddend , scraps and the palette helped designers in collaborating with one another (design behaviors 12 -- 14). Intentional interfaces helped designers in moving between synchronous work and opportunistically branching off to asynchronous work. The fading highlighter enabled designers to explain their sketches to one another from remote computers. The palette further allowed designers to bring work together from different canvases, and scraps also help to coordinate bringing work together.


%Analysis of grid versus intentional interfaces. We reduced the number of features that used to be there where each serve multiple purposes.

%\section{Limitations and threats to validity}

%--> What do these sessions allow me to say? The sessions are not directly comparable.

%
%In this section, I address potential issues which may lead to threats in validity.
%
%- Was not able to observe studies in person.
%- People interviewed may not have remembered correctly, or incorrectly interpreted logs.
%- All uses of Calico may have been idiosyncratic, and specific to the culture observed.
%- Observed different phases of the lifecycle for each group.

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 
 \newpage \chapter{Related Work}
\label{chapter:related-work}

Existing sketch tools can be classified into two categories: (1) those that interpret sketches with the purpose of turning them into formal objects, and (2) those that support sketching activities in general. The survey on sketch-based systems by Johnson et. al. \citep{Johnson} provides a more extensive look of all tools created to date than what we necessarily can provide in the blow.

\section{Tools that Interpret}
\label{relatedwork:1}

The  tools within this category interpret a digital freeform sketch and then generate a formal representation. While we purposefully chose not to interpret sketches in Calico, these tools remain relevant as background work.

Many tools that interpret sketches focus on generating prototypes for user interfaces. One of the earliest  such tools is SILK \citep{Landay}, which allowed users to generate a usable GUI from a sketch, and then generate the corresponding Java code to implement that GUI. SILK was the first such tool of its kind, which stressed both the importance of working in a low-fidelity environment, but enabled rapid iterations of mockups by generating usable simulations to give near immediate feedback. SILK differed greatly from Calico in that it helped designers iterate by providing feedback, while Calico encourages developers by making sketches easier to manipulate using scraps. Another tool, DENIM \citep{newman2003denim}, allows users to create low fidelity mockups of websites and then run simulations of that website. DENIM built on the concepts of SILK by both supporting interpretation of sketched content and navigation between webpages. DENIM contributed the notation of multiple levels of hierarchy for navigating sketches, in which the user zoomed out to navigate the network of websites, and zoomed in to view sketched content in webpages. DENIM's use of multiple levels of levels of abstraction for navigating webpages, and linking webpages, is similar to Calico's intentional interfaces, but is highly specialized for designing webpages. Other tools targeted multiple domains, such as the tools InkKit and Freeform, which created usable software systems from sketches \citep{chung2005inkkit,Plimmer}. These family of tools stressed the importance of retaining the sketchy look-and-feel of sketches, which was based on their insight that visually beautifying content too early can be harmful to the designer \citep{Shipman}. Unlike previous tools, InkKit and Freeform interpreted different types of sketches, such as user interface and ER diagrams, which they used to generate a working prototype system. By interpreting sketches from multiple perspectives, these systems provide rich prototypes for feedback at a low cost of effort for the designer. These systems are similar to Calico in that users can sketch a system from multiple perspectives, however these tools restrict users to using a pre-determined set of notations.

Another group of interpretive sketch systems includes those that recognize UML elements. The earliest such tool was Knight \citep{damm2000tool}, which allowed  designers to create partial UML diagrams by selecting a portion or all of their sketch to be interpreted as UML elements. Knight was inspired by studying software designers at the whiteboard, and contributed the observation that software sketches begin vague, but become refined over time. The Knight tool allowed designers to sketch, and covert their sketches into a beautified UML representation by manually triggering it. Both Calico and Knight allow users to refine their drawings from plain whiteboard sketches into objects at a later point, however Knight only interprets UML class and sequence diagram notations. The tool SUMLOW \citep{chen2008sumlow} further advanced on the work of Knight by allowing interpreted sketches to retain their sketchy appearance after interpretation (citing InkKit as inspiration), and allowed the mixing of different UML notations. While both Knight and SUMLOW made UML diagrams a less viscous representations by allowing users to simply sketch additions, the retainment of the sketchy appearance after interpetation introduced another step of provisionality to sketches, which was not available to software designers in other sketch interpretation tool. Both Calico and SUMLOW are similar in this philosophy. Hosking and Grundy later integrated this tool into a full software design environment, Marama-Sketch \citep{Grundy}, which designers can use to sketch diagrams in a broad variety of notations. Marama sketch brought tighter integration between sketched models and actual code, a quality which Calico itself does not have. Numerous other UML-oriented sketch tools have been created, generally following similar strategies (for a survey, see \citep{Johnson}). 

Another approach taken was building a general purpose framework for interpreting sketches. LADDER was such a tool that provided an extensible, general purpose language which could be adapted to many notations, including software design notations, biology and chemistry notations, and military symbols using maps \cite{hammond2006ladder}. Of the many approaches, most target a single, or small set of notations. Dixon's work provided an approach that attempted to unify different interpretation engines by providing a framework that queries multiple engines simultaneously, and ranks the results by their probability of success \cite{dixon2008whiteboards}. These tools vary greatly from Calico given that they are domain specific, while Calico remains a domain-generic environment.

Other tools allow mixed elements to coexist in the same area as well, though interpretation is left to explicit manual choice by the user rather than automated recognition. An example of such a system is Patches, in which the user can create sets of irregular shapes called ``patches'' \citep{Kramer}. These can be moved around, be made translucent, and, most importantly, the patch can be assigned an interpretation where it can become a list, outline, table, or other element. The functionality, as well as look-and-feel, of the scraps functionality in Calico is fundementally very similar to the Patches system. However, they differ in their goal, where Patches viewed its functionality as providing lenses over existing sketches, and provided transparencies for annotations over existing sketches such as architecture diagrams. Calico, on the other hand, uses scraps as building blocks to build representations within the domain of software. Kramer's work later inspired the overall functionality of the sketch framework SATIN \citep{Hong}, which was later used to create the tools DENIM (previously mentioned) and Designer's Outpost (mentioned in the next section). 

\section{Systems that Support Sketching Activities}
\label{relatedwork:2}

The second type of sketch system is aimed at supporting the general design activity. These systems help organize sketched artifacts during meetings, make them retrievable, and help manage and maintain the plethora of sketched design content that gets generated during design sessions. 

The very first of these systems were created for the Liveboard system at PARC. Colab was the first system to work with this hardware, and acted as a meeting tool \citep{Stefik}. It had a simple, uncluttered interface with a mode selector and a visible set of saved sessions in the fashion of a filmstrip. The next system developed for the Liveboard was Tivoli \citep{Pederson}, which used the filmstrip metaphor to manage sketches in a side panel. Tivoli also organized sketches into chunked items for easy moving, and had additional functionality for creating empty space. Tivoli did not attempt to recognize any of the drawn elements, but rather provided intelligent support for common tasks, such as creating lists and checking off items in lists. The clustering of items and use of lists in Tivoli is similar to the functionality that scraps and list scraps provide. After Tivoli, Dolphin \citep{Streitz:1994:DIM:192844.193044} introduced a client-server architecture for group collaboration and allowed users to create links between canvases. Dolphin was the first whitebaord system to provide hyperlinks between content in canvases, which would not appear again in other tools until the creation of DENIM ten years later. Hyperlinks in Dolphin provide more flexibility than intentional interfces for organizing lists, but Dolphin did not provide a visualization for the organizing of canvases, such as Calico's cluster view. A significant difference between these systems and Calico lies in intentional interfaces, which provides a spatial orientation of sketches as compared to the paginated style, which leads to older sketches moving around. 

Many other tools provide methods for managing the presence of multiple sketches within a fixed space. PostBrainStorm \citep{guimbretiere2001fluid} uses the border areas as areas where sketches auto-shrink, thereby minimizing space use by elements that are temporarily set aside. PostBrainstorm presented an exploration of an ubiqutious computing approach to whiteboard software and hardware, allowing users to place a physical picture on a whiteboard, and have the same picture automatically appear in the system. While Calico did not experiment in the domain of ubiquitous computing, the importing of existing photos and documents were important features in the use of Calico as well. Flatland \citep{mynatt1999flatland} implemented a variant of this last idea by automatically grouping sketches into clusters, and shrinking and moving them out of the way when more space is needed. Flatland was not the first to provide clusters, but Flatland contributed a rich exploration of it's capability for specialized support, including organizing lists, creating maps, etc., and allowing users to move backwards and forwards in the history of clusters. Scraps within Calico, in comparison, do not support as many domains, but instead are stackable and relate-able. Range \citep{Ju} extended the approach taken by Flatland by using a person's physical proximity to the board as a trigger to create space, and also presented past sketches back to the designer when they were physically far away from the board in order to provide inspiration for creative work. Overall, many of these systems attempt to support the designer at the whiteboard in general activities such as clustering objects and making space without the designer manually requesting this space. Calico does not provide such ubiquitous support in a single whiteboard, but instead Calico allows users to manually organize their sketches using intentional interfaces because manually organizing spaces has value in one's own design process.

A third set of tools in this second type specifically target the organization of multiple sketches and canvases. Bellamy et al. \cite{Bellamy:2011:STI:1985793.1985909} presented an ideation tool for building user interfaces. Users could sketch the states of a user interface, and the sketch presented a set of storyboards back to the user. Additionally, the history of the user's exploration was presented back to the user as a tree that traced the undo history. Other systems targeted the visualization of separate canvases, such as IdeaVis \cite{geyer2012ideavis}, which presented a visualization for a network of whiteboard canvases. The IdeaVis system captures hand-drawn sketches using a camera, and presents them back to a co-located sketch team using an ``interactive hyperbolic tree visualization'', in which users can navigate a network of canvases, and the visual size of canvases become larger when in close proximity to the active canvas. Their studies showed that automatically making relevant canvases large helped users maintain focus to important sketches. This system is most similar to Calico's intentional interfaces in that they both visualize a network of canvases, however intentional interfaces maintains a fixed size for canvases.

Another set of tools targeted the collaboration aspect of electronic whiteboard systems. Designer's Outpost \citep{klemmer2001designers} bridged the gap between physical Post-It notes and digital content, transitioning content of Post-It notes into digital artifacts that then could be organized in various ways. The core functionality of this system allowed users to create links between digital Post-It notes, group Post-It notes into clusters, and zoom in and expand individual Post-It notes to write in great detail. The designer's outpost system provided novel methods of interacting with remote users, such as as showing a user's shadow on the whiteboard in order to communicate gestures and body language, and using a pointer. Calico, in comparison, does not communicate body language, but instead uses the fading highlighter to communicate transient information. TEAM STORM \citep{Hailpern} presented another approach, which addressed the issue of public and private spaces. Users could create sketches on their own devices, and had the option of sharing their sketches in the public space, in which other users could interact with and provide feedback on sketches. Further building on the notation of a shared collaborative space, the GAMBIT \cite{Sangiorgi:2012:GAM:2305484.2305527} system addressed the issue of using multiple devices together, including large displaces, laptops, tabletops, and phones. In studies of GAMBIT, researchers observed a trend in sessions in which users first build mental models of their domain, construct scenarios, and lastly sketch prototypes using GAMBIT. The notation of public and private spaces using GAMBIT was viewed as helpful to users. In comparison, sketches in Calico are always public, but users can move to a separate canvas to work asynchronously. Lastly, Camel \cite{cataldo2009camel} integrated a whiteboard system into a formal Eclipse UML environment, allowing users to post UML diagrams onto poster boards. By integrating into a formal environment, Camel provided software teams value by supporting code reviews and design walkthroughs which were very closely tied to a team's software code. 

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 
 \newpage \chapter{Conclusions and Future Work}
\label{chapter:conclusions}


In this dissertation, I have presented a sketch-based environment to support software designers during the early phases of design. I have done so by building a set of minimally invasive features that together support the set of fourteen design behaviors by software designers. Specifically, designers across a range of disciplines used scraps to create many different types of representations in their work. They used the grid and intentional interfaces to navigate between their sketches, and, importantly, intentional interfaces helped designers to recall the context in which they were created. Lastly, they used intentional interfaces and the fading highlighter to collaborate within their team, both in local and distributed settings.

My analysis evaluated the design sessions of 39 individuals, comprising of 16 computer science graduate students at the whiteboard, 16 computer science graduate students using Calico Version One, and 7 professionals across three software-related organizations using Calico Version Two. From these design sessions, 1) I have built and tested an initial prototype, Calico Version One, and 2) built the second iteration, Calico Version Two, and performed a field evaluation at three different professional organizations using Calico to conduct their own work. From these deployments, I have gained insight into how each feature supported the design behaviors in practice.

Scraps were a powerful feature that played a major role in supporting the kinds of sketches that designers create, and a minor role in supporting them to navigate and collaborate. Scraps were most useful in representing box-and-arrow type software representations in the OSS group and the research group. Most scraps used little detail, most times only having the name of the component they represented. The improvised notations were surprising as well, in which users tagged scraps and connectors with colors to signify meaning in their design sessions. Scraps also served to help navigate between sketches in the same canvas by helping to juxtapose diagrams, and move diagrams out of the way. Overall, scraps did not satisfy everyone's needs, such as the interaction designer group, but in most cases, they were helpful in supporting many of the design behaviors in practice.

The palette played a minor role in supporting the design behaviors, only supporting the use of impromptu notations and juxtaposing diagrams. While I expected designers to use the palette more in practice, designers preferred to reuse diagrams simply by redrawing them by hand, using the scrap copy feature, or copying a canvas. However, the scrap served the role as a global clipboard, and while it was very rarely used, it served an important purpose: to easily move content across canvases, which designers did to reuse particular sketches as a reference.

Intentional interfaces played a large role in supporting navigation between sketches and collaboration. It gave all groups that used it a sense of boundless space that encouraged them to sketch more, and which they used to create different perspectives, alternatives, and levels of abstraction. Clusters provided a means to separate projects, and canvases linked using tagging provided a means to relate canvases to one another. Most importantly, linked canvases provided an ordering to canvases, which users in the OSS group and the research used to create a narrative from their sessions. The organization of intentional interfaces further served as a useful metaphor to organize collaboration, which occurred in both co-located and distributed settings.

The fading highlighter only supported two design behaviors, mental simulations and explaining designs, but it supported these design behaviors in ways that the other features could not. The fading highlighter was sometimes not used because users forgot that it was there, but when it was used, it served as an aid to verbal walkthroughs. It was more helpful than a simple pointer because the speaker could draw arrows, and could do so without modifying the original sketch. In particular, it was most helpful in multi-device or distributed settings, allowing listeners to look to the large electronic whiteboard while another person sketched from their device. 

Together these four features supported all fourteen design behaviors, and did so while being minimally invasive to the design activity. The features worked together to support the design behaviors, and did so while maintaining the flexibility and fluidity of the regular whiteboard. 

\section{Summary}

My contribution from each of the chapters are as follows:

 \begin{enumerate}

   \item \textbf{Chapter \ref{chapter:motivation} (\nameref{chapter:motivation})} introduced the definition of design behaviors in detail and presented the grand set of design behaviors that I aimed to support. 

   \item \textbf{Chapter \ref{chapter:research-question} (\nameref{chapter:research-question})} posed the topic that this dissertation addressed: \textit{What minimally invasive, coherent set of features can be design that is sufficient to effectively support these behaviors?} I layed out the method by which I approach this topic and how I was to evaluate this approach.

   \item \textbf{Chapter \ref{chapter:calico-version-one} (\nameref{chapter:calico-version-one})} presented the first prototype of Calico, which was built toward testing the feasibility of the approach. It did so by first testing a subset of the design behaviors, specifically design behaviors 1, 2, 3, and 5. To address these behaviors, I introduced three features: scraps, the grid, and the palette. The approach was evaluated in a comparative study between Calico and the regular whiteboard, both with respect to the design behaviors performed as well as the design conversations that took place. In the qualitative analysis, I found that the four design behaviors were indeed supported by Calico, however, participants did not feel they were well supported by scraps. From the quantitative conversation analysis, I found that the design activity was carried out the same in Calico as it is on the whiteboard, and Calico may lead to more shared focus between participants and longer discussions of individual topics.

   \item \textbf{Chapter \ref{chapter:calico-version-two} (\nameref{chapter:calico-version-two})} introduced the second and final version of Calico in this chapter. These features were: the revised interaction for scraps, the intentional interfaces features that replaces the grid, the distributed nature of the architecture, as well as other features targeted at supporting the full set of design behaviors. I further present their implementation in detail, present examples of how these features support the fourteen design behaviors, and describe software implementation of this second iteration.

   \item \textbf{Chapter \ref{chapter:evaluation} (\nameref{chapter:evaluation})} presented the study of the final version of Calico ``in the wild'' at a commercial open source software company, an interaction design company, and a software research group. Each setting presented a unique set of representations, methods for navigations over sketches, and ways of collaborating. In the commercial open source software company, Calico supported a group of developers in designing the next iteration of their software in both individual and group design sessions. In the interaction design group, Calico supported a pair of designers processing a set of interviews to create personas in preparation for a software project. In the research group, Calico supported a distributed group collaborating in building a software system and also helped to onboarding a new member.

   \item \textbf{Chapter \ref{chapter:discussion} (\nameref{chapter:discussion})} took a step back and put my work in a broader context. I discussed the implications of my findings, reconnected my work to the broader design literature, and provided some findings and observations that are outside the evaluative framework of the design behaviors, but are worthwhile to highlight nonetheless. 

% discussed the implications of my findings, reconnect my work back to the research question in Chapter \ref{chapter:research-question}. From the interviews, I found that Calico was minimally invasive to those working with software representations such as box-and-arrow and state diagrams. However, the scraps feature and its system gestures was found to be confusing for those with existing experience with other sketch tools that have their own set of gestures. Most features within the tool were found to be cohesive with one another, which was judged by the number of features that support one another. Over half of the features were benefited by the other features, where basic sketching received the most benefit from others, and the fading highlighter received the least benefit. 

%   With respect to the design behaviors, all design design behaviors were observed within the three groups, and were supported in doing so with Calico's features. All groups used scraps to create different types of representations that were low-detailed, were refined over time, and used improvised notations. Two of the three groups, the OSS group and the research group, used intentional interfaces to navigate between their canvases, moving between different levels of abstraction, perspectives, and alternatives. They also used the fading highlighter to review their work and mentally simulated over their sketches in group meetings. The same two groups used intentional interfaces and the fading highlighter to collaborate over their sketches by explaining sketches and diverging into asynchronous work. The groups, however, did not bring their work back together, as the final design behavior suggested.

   \item \textbf{Chapter \ref{chapter:related-work} (\nameref{chapter:related-work})} reviewed the existing literature of tools that support software design on an electronic whiteboard, covering both tools that support sketch recognition and those that support the management of sketches.
 \end{enumerate}

\section{Directions for Future Work}
This dissertation illuminates several potential areas of study for future work. 

\subsection{Compositional notations}

The first area of future work extends from providing more domain specific support for scraps. While each group did not strictly use notations, they did create sketches that borrowed idiomatic pieces from various formalisms, such as UML class diagrams, process flow diagrams, etc. In many cases groups used scraps to represent these notations, such as when the OSS group used boxes-and-arrows and the research group used state diagrams, but scraps fell short in fully supporting these custom additions. For example, both groups added cardinality to connectors using plain sketching, but the cardinality was left behind when the scrap was moved. There is an opportunity in this case, and many others, to provide formal support for these elements. 

There are many ways to support these elements. One such case would be to change a scrap itself into specific types of scraps, such as a ``UML class diagram scrap'' or a ``user interface scrap''. List scraps is an example of such as approach, in which an existing scrap can be changed into list scrap to vertically order a set of items. However, this approach does not fully take into account the behavior of refinement of sketches observed in design behavior 3, and may prematurely commit a designer to using a formal notation, when they only want to borrow a part of that notation. This leads to the second approach, which is to allow designers to add parts of notations to existing scraps. I call this approach of adding parts of notations to scraps ``compositional notations'', in which a scrap may be ``composed of'' notational elements, such as a UML class diagram cardinality element, or a user interface element. By allowing the user to build the level of formality of their scrap piece-by-piece, the user can use only those elements that they need, and refine the appearance and behavior of their scraps over a period of time, without enforcing all of the rules of the eventual formal notation too early. 

The compositional notations feature is based on the observation that many formal notations, despite being different, share some parts of their notations. For instance, certain arrows are universally regarded as connectors in UML, Entity-Relationship, and activity diagrams, to name a few. As another example, shape identifies the type of element in both ER Chen notation and activity diagrams. My intuition, then, is to make available these shared, elementary parts as building blocks, so that they can be composed into
notations when and where the designer wants. 

%Based on my observations that designers in various situations do not begin at a notation, but rather refine a sketch by adding notational elements to it (design behavior 3), designers would be better served if they can build on the existing scraps they use.  
%
%Calico current supports this to a degree.
%I have made such a suggestion in a past position paper, which calls this refined support \textit{software modes}
%
%- lists
%- box and arrow diagrams
%- state diagrams
%- tables
%- user interfaces
%////////////
%- designing software
%- designing user interfaces
%- designing requirements
%///
%position paper

\subsection{Informal analysis of scraps}
%
%The second way to support is to provide lightweight feedback of the sket
%
%The second way is to provide an analysis of 
%
%Providing analysis over scraps... nick and alfredo paper
%
A second path of future work is to explore the automated feedback that could be given to a design in an informal setting. The obstacle in this case is that sketching does not lend itself to automated feedback as a sketch is typically not interpretable by anyone other than the designer(s) themselves. In the case of Calico, however, there is the opportunity to build on the scraps feature to bring feedback that is automatically provided by the environment. The key choice, however, is which kind of feedback. In the face of sketching not to be interrupted, it is clear that formal models should be ruled out -- they require far too much specification before they can provide useful feedback; they simply are best left for later phases of modeling. At the same time, one should ask whether the precision that typically comes with the formal models is necessary. More specifically, scraps could be assigned imprecise annotations that enable some interpretation of a sketch.

The hypothesis underneath this approach is that precise values and specifications are not necessary. Rather, I postulate that during early design it is necessary to have an idea of what might happen, to have an approximate answer, and that it is necessary to quickly vary the underlying parameters from which this approximation is derived such that the qualities that are sought can be explored with. 
This approach was investigated in more depth within a workshop paper that provided such support by building on top of Calico \cite{mottalightweight}. It posed four questions:

\begin{enumerate}
	\item What kind of analysis do we want to exploit? 
	\item What type of model is the user going to draw? 
	\item How will the tool acquire the necessary inputs for the models analysis? 
	\item How is the feedback presented to the user?
\end{enumerate}

While the research in the workshop paper did not investigate these questions in detail, it provided an initial prototype to address these questions. It presented a prototype built as a plugin to Calico. It utilized performance analysis because the domain is relatively simple in comparison to other formal approaches. It used activity diagrams, as opposed to other notations such as sequence diagrams, to model performance because past work showed that designers discuss message passing between software components by pointing at representations of the software components and entities. In order to acquire inputs to build the activity diagram, scraps could be tagged with generic values, such as CPU process, network event, database access, which provided generic time-based values for analysis. The feedback was presented back to the user by changing the opacity of the scraps, where the elements in a slow performing diagram became more transparent towards the end of the path.

%in which a user could sketch an informal activity diagram in Calico and receive have their activity diagram change opacity to reflect if the system described by the activity diagram was slow . This work posed 
%
%- research group may have benefited from logic checking in state diagrams
%- OSS group may have benefited from a performance analysis.
- 

\subsection{Contextualized sketching}

A third path of extending work is to embed outside information within Calico so that it can be explored in an informal setting. This area of research is motivated by the observation that much design takes place in the context of a system already under development [1-2], with the code base in existence taken into account. The sketches created during such maintenance design sessions often explicitly represent the existing code base in an abstract manner. Sometimes, as was the case in some of the design sessions of the OSS group, the sketch is even as simple as a few unlabeled boxes that verbally are identified, other times, the sketch contains a significant amount of detail duplicating the code base. In such cases, valuable time can be wasted drawing these diagrams, and there is always the risk that the diagram that is drawn does not faithfully represent the code as it currently exists.

To support these kinds of sessions, Calico could be extended to create a sketchy representation of a code base that can be interactively explored, manipulated, and augmented by designers. An initial idea is that a developer, directly in Calico, should be able to point to a configuration management repository, ask for a specific version of the code for one file, and then be presented with a scrap that represents that file and a set of outgoing lines to other scraps that represent code in the proximity of the main file under consideration. This set of scraps is augmented with a set of explosion points, small markers that when tapped expand the information available to the developers and let them navigate the code base (not unlike Code Bubbles [65]). Tapping on one of the auxiliary classes, for instance, might make it the center and bring up its connections. Tapping a class itself may bring up its methods, tapping a method might even bring up its code. This is of course but one way in which code could be represented. Other ideas involve reverse engineering techniques that automatically identify key methods in classes and just bring those up, using a zooming and panning technique like Code Canvas [66], using slicing to quickly bring up related code, and using higher levels of abstraction such a components or services.

\subsection{Design histories}

A fourth way of extending Calico is to provide a visualization of the design history. The usage logs generated by Calico can be used to reflect on one's own design history, as well as study the design process of professional designers. In addition to these, there are two further opportunities to extend this research, the first of which explores knowledge captured by tags in intentional interfaces, and the second of which juxtaposes views to compare design sessions.

In the first case, the tags captured within intentional interfaces can be used to build a more comprehensive design history. From the history of navigation alone, the usage logs can be used to create time-ordered sequences of the design changes. With tags from intentional interfaces, the time-order sequences can be annotated to highlight important events, such as when an alternative was explored, when designers moved levels of abstraction, and other events as well. 

In the second case, designs histories could be visualized in such a fashion that they can be compared. For instance, the nature of early sessions where the basis for a design is laid could be compared against later sessions where this basis is refined. In another case, this could be used in education to review the design sessions of students. In this latter case, a professor may want to review the design histories of students for homework, but reviewing the design histories of thirty or more students may not be feasible. Instead, the professor may benefit from a visualization that juxtaposed the design sessions of all students in the class so that general trends appear from the histories, for example, if the majority students fixated on one aspect of their design, or did not sufficiently explore alternative solutions.

Capturing design histories by reviewing the usage logs would allow individuals to reflect on the overall design process taken. Previous research has shown that novice and professional designs differ in how they arrive at a solution. For example, novice designs are more likely to fixate on a single problem, while professionals will provisionally make a decision and move on [cite]. Similarly, professionals may focus on two subjects at a time, and regularly rotate between pairs of subjects \cite{Baker2010590}. 
Calico can take advantage of this in two ways: 1) recording the process taken by professional designs in the field so that it can be studied, 2) recording the process of students so that the professor can provide feedback to the students.

With respect to recording the design process of professionals, a professor may use the recorded process as an examplar of official design. In past design courses, professors have provided students with case studies of professionals conducting design, such as videos of them at work. Selected portions of these design histories could be given to the students by the professors as examples. Previous proposals of tools have proposed similar methods of reviewing design histories, such as the Design practice stream (DPS) tools by Nakakoji \cite{Nakakoji6035659}, which plays back a history, and CogSketch \cite{Forbus1149}, which is used to understand how glyphs are used across all several domains.

With respect to recording the design process of students, Calico would enable professors to observe the design activity of students while outside of class. While professors can step in and provide guidance to students who become stuck in class, much work happens outside of class where they cannot provide guidance. The professors often can only see the final artifact or document that students submit. By having histories available, professors can review the histories to see if students are fixating on problems, adequately exploring alternatives, and so on.

It should be noted that the tools to review the usage logs generated by Calico are rudimentary. They provide a usage summary, as well as a set of screenshots of all actions taken, which can become highly verbose and be time consuming to review. There is an opportunity to improve this process so that professors and students can more effectively review their logs.

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 
% \newpage \chapter{Future Work}
\label{chapter:future-work}


\section{Background}

Lorem ipsum 

%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 
% ... and so on

% These commands fix an odd problem in which the bibliography line
% of the Table of Contents shows the wrong page number.
\clearpage
\phantomsection

% "References should be formatted in style most common in discipline",
% abbrv is only a suggestion.
\bibliographystyle{abbrv}
\bibliography{thesis}

% The Thesis Manual says not to include appendix figures and tables in
% the List of Figures and Tables, respectively, so these commands from
% the caption package turn it off from this point onwards. If needed,
% it can be re-enabled later (using list=yes argument).
\captionsetup[figure]{list=no}
\captionsetup[table]{list=no}

% If you have an appendix, it should come after the references.
 \newpage % The original template (from Trevor) had a custom \appendix command,
% but I found it to break figure/table counters. I'm not sure how
% reliable my fix is, so I ended up reverting back to the standard
% latex version, and renaming the custom command to \myappendix.  You
% can try both and see how things work out:
% 1) Call \appendix once, and then make each appendix a \chapter
% 2) Call \myappendix once, and then make each appendix a \section.

\appendix
\chapter{Appendix Title}

Supplementary material goes here. See for instance Figure
\ref{fig:quote}.

\section{Lorem Ipsum}

dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
commodo consequat. Duis aute irure dolor in reprehenderit in voluptate
velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint
occaecat cupidatat non proident, sunt in culpa qui officia deserunt
mollit anim id est laborum.

\begin{figure}
  \centering
  \begin{tabular}{l}
    ``I am glad I was up so late,\\
    \quad{}for that's the reason I was up so early.''\\
    \em \footnotesize William Shakespeare (1564-1616), British
    dramatist, poet.\\
    \em \footnotesize Cloten, in Cymbeline, act 2, sc. 3, l. 33-4.
  \end{tabular}
  \caption{A deep quote.}
  \label{fig:quote}
\end{figure}


%%% Local Variables: ***
%%% mode: latex ***
%%% TeX-master: "thesis.tex" ***
%%% End: ***
 \newpage 

\end{document}