@conference {18328,
	title = {A case for query by image and text content: searching computer help using screenshots and keywords},
	booktitle = {Proceedings of the 20th international conference on World wide web},
	series = {WWW {\textquoteright}11},
	year = {2011},
	month = {2011///},
	pages = {775 - 784},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {The multimedia information retrieval community has dedicated extensive research effort to the problem of content-based image retrieval (CBIR). However, these systems find their main limitation in the difficulty of creating pictorial queries. As a result, few systems offer the option of querying by visual examples, and rely on automatic concept detection and tagging techniques to provide support for searching visual content using textual queries.This paper proposes and studies a practical multimodal web search scenario, where CBIR fits intuitively to improve the retrieval of rich information queries. Many online articles contain useful know-how knowledge about computer applications. These articles tend to be richly illustrated by screenshots. We present a system to search for such software know-how articles that leverages the visual correspondences between screenshots. Users can naturally create pictorial queries simply by taking a screenshot of the application to retrieve a list of articles containing a matching screenshot.
We build a prototype comprising 150k articles that are classified into walkthrough, book, gallery, and general categories, and provide a comprehensive evaluation of this system, focusing on technical (accuracy of CBIR techniques) and usability (perceived system usefulness) aspects. We also consider the study of added value features of such a visual-supported search, including the ability to perform cross-lingual queries. We find that the system is able to retrieve matching screenshots for a wide variety of programs, across language boundaries, and provide subjectively more useful results than keyword-based web and image search engines.
},
	keywords = {content-based image retrieval, multimodal search, online help},
	isbn = {978-1-4503-0632-4},
	doi = {10.1145/1963405.1963513},
	url = {http://doi.acm.org/10.1145/1963405.1963513},
	author = {Tom Yeh and White,Brandyn and San Pedro,Jose and Katz,Boriz and Davis, Larry S.}
}
@conference {13080,
	title = {A case for query by image and text content: searching computer help using screenshots and keywords},
	booktitle = {Proceedings of the 20th international conference on World wide web},
	series = {WWW {\textquoteright}11},
	year = {2011},
	month = {2011///},
	pages = {775 - 784},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {The multimedia information retrieval community has dedicated extensive research effort to the problem of content-based image retrieval (CBIR). However, these systems find their main limitation in the difficulty of creating pictorial queries. As a result, few systems offer the option of querying by visual examples, and rely on automatic concept detection and tagging techniques to provide support for searching visual content using textual queries. This paper proposes and studies a practical multimodal web search scenario, where CBIR fits intuitively to improve the retrieval of rich information queries. Many online articles contain useful know-how knowledge about computer applications. These articles tend to be richly illustrated by screenshots. We present a system to search for such software know-how articles that leverages the visual correspondences between screenshots. Users can naturally create pictorial queries simply by taking a screenshot of the application to retrieve a list of articles containing a matching screenshot. We build a prototype comprising 150k articles that are classified into walkthrough, book, gallery, and general categories, and provide a comprehensive evaluation of this system, focusing on technical (accuracy of CBIR techniques) and usability (perceived system usefulness) aspects. We also consider the study of added value features of such a visual-supported search, including the ability to perform cross-lingual queries. We find that the system is able to retrieve matching screenshots for a wide variety of programs, across language boundaries, and provide subjectively more useful results than keyword-based web and image search engines.},
	keywords = {content-based image retrieval, multimodal search, online help},
	isbn = {978-1-4503-0632-4},
	doi = {10.1145/1963405.1963513},
	url = {http://doi.acm.org/10.1145/1963405.1963513},
	author = {Tom Yeh and White,Brandyn and San Pedro,Jose and Katz,Boriz and Davis, Larry S.}
}
@conference {18352,
	title = {Searching documentation using text, OCR, and image},
	booktitle = {Proceedings of the 32nd international ACM SIGIR conference on Research and development in information retrieval},
	series = {SIGIR {\textquoteright}09},
	year = {2009},
	month = {2009///},
	pages = {776 - 777},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {We describe a mixed-modality method to index and search software documentation in three ways: plain text, OCR text of embedded figures, and visual features of these figures. Using a corpus of 102 computer books with a total of 62,943 pages and 75,800 figures, we empirically demonstrate that our method achieves better precision/recall than do alternatives based on single modalities.},
	keywords = {Computer vision, content-based image retrieval, multimodal search},
	isbn = {978-1-60558-483-6},
	doi = {10.1145/1571941.1572123},
	url = {http://doi.acm.org/10.1145/1571941.1572123},
	author = {Tom Yeh and Katz,Boris}
}
@conference {18329,
	title = {A picture is worth a thousand keywords: image-based object search on a mobile platform},
	booktitle = {CHI {\textquoteright}05 extended abstracts on Human factors in computing systems},
	series = {CHI EA {\textquoteright}05},
	year = {2005},
	month = {2005///},
	pages = {2025 - 2028},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {Finding information based on an object{\textquoteright}s visual appearance is useful when specific keywords for the object are not known. We have developed a mobile image-based search system that takes images of objects as queries and finds relevant web pages by matching them to similar images on the web. Image-based search works well when matching full scenes, such as images of buildings or landmarks, and for matching objects when the boundary of the object in the image is available. We demonstrate the effectiveness of a simple interactive paradigm for obtaining a segmented object boundary, and show how a shape-based image matching algorithm can use the object outline to find similar images on the web.},
	keywords = {content-based image retrieval, interactive segmentation, mobile interface, Object recognition},
	isbn = {1-59593-002-7},
	doi = {10.1145/1056808.1057083},
	url = {http://doi.acm.org/10.1145/1056808.1057083},
	author = {Tom Yeh and Grauman,Kristen and Tollmar,Konrad and Darrell,Trevor}
}
@conference {18344,
	title = {IDeixis: image-based Deixis for finding location-based information},
	booktitle = {CHI {\textquoteright}04 extended abstracts on Human factors in computing systems},
	series = {CHI EA {\textquoteright}04},
	year = {2004},
	month = {2004///},
	pages = {781 - 782},
	publisher = {ACM},
	organization = {ACM},
	address = {New York, NY, USA},
	abstract = {We demonstrate an image-based approach to specifying location and finding location-based information from camera-equipped mobile devices. We introduce a point-by-photograph paradigm, where users can specify a location simply by taking pictures. Our technique uses content-based image retrieval methods to search the web or other databases for matching images and their source pages to find relevant location-based information. In contrast to conventional approaches to location detection, our method can refer to distant locations and does not require any physical infrastructure beyond mobile internet service. We have developed a prototype on a camera phone and conducted user studies to demonstrate the efficacy of our approach compared to other alternatives.},
	keywords = {content-based image retrieval, context-aware computing, location-awareness, Mobile computing},
	isbn = {1-58113-703-6},
	doi = {10.1145/985921.985933},
	url = {http://doi.acm.org/10.1145/985921.985933},
	author = {Tom Yeh and Tollmar,Konrad and Darrell,Trevor}
}
