www.buildwindows.com

Bring pen and touch input to your Metro style apps with inkAnnie Chowdhry

Program ManagerMicrosoft Corporation

PLAT-192C

Jay PittmanDeveloperMicrosoft Corporation

www.buildwindows.com

Metro style apps in Windows 8 are so much better with Ink!

Code for touch, get mouse and pen for free

www.buildwindows.com

Agenda

• How Inking Works• Inking API architecture • Demos• Creating and rendering Ink to a canvas• Managing Ink from pen and touch input • Converting Ink to text (handwriting recognition)

You’ll leave with examples of how to• Write an Ink-enabled Metro style app in Windows 8

www.buildwindows.com

Ink Pad AppOverview of Inking – capture, manipulate, and recognize

demo

www.buildwindows.com

Recognize

Capture and Manipulate

How Inking works

X,Y….Xn,Yn

Optional

Pointer API captures pen motion, passing coordinates to Ink API

Ink API helps render and stores motion as ink. Ink manipulation.

Ink API groups strokes and passes them to recognizer

Reco API returns interpreted combination of strokes from recognizer (rich recognition results)

l HI

!

HI

!

HI! Hi! Hit Hid hi!

www.buildwindows.com

Metro style app

Inking API architecture – execution flow

Windows.UI.Input.

Inking

stroke builder,rendering helper,ink manipulation,

handwriting recognition with rich recognition results

with alernates,clipboard,

serialization

Input Render

Obtains Raw Data

position, id and pressure using

MSPointer (JavaScript / ICoreWindow (XAML)

and passes this to stroke builder:

Begin/Append/End Stroke methods

Renders Strokes

render using rendering helper method:

InkRenderingSegment

does not return packets but stroke

rendering segments such as Bezier curves

Code Snippets• Register for events• ProcessPointerDown• ProcessPointerUpdate• ProcessPointerUp• Render everything

Register for events

html:    <canvas id="myCanvas" width="1000" height="1000"></canvas> 

js:var canvas = document.getElementById("myCanvas");var context = canvas.getContext("2d"); canvas.addEventListener("MSPointerDown", handlePointerDown, false);canvas.addEventListener("MSPointerMove", handlePointerMove, false);canvas.addEventListener("MSPointerUp", handlePointerUp, false); 

var inkManager = new Windows.UI.Input.Inking.InkManager();

ProcessPointerDownfunction handlePointerDown(evt){    if ((evt.pointerType === 3) || ((evt.pointerType === 4) && (evt.button === 1)))    { evt.preventManipulation();        

inkManager.processPointerDown(evt.currentPoint);         context.beginPath();

context.moveTo(evt.currentPoint.rawPosition.x, evt.currentPoint.rawPosition.y);    } else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { // Handle touch down }}

ProcessPointerUpdate

function handlePointerMove(evt){    if ((evt.pointerType === 3) || ((evt.pointerType === 4) && (evt.button === 1)))    { evt.preventManipulation();

        inkManager.processPointerUpdate(evt.currentPoint);

       context.lineTo(evt.currentPoint.rawPosition.x, evt.currentPoint.rawPosition.y);       context.stroke();    } else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { // Handle touch move }}

ProcessPointerUpfunction handlePointerUp(evt){    if ((evt.pointerType === 3) || ((evt.pointerType === 4) && (evt.button === 1)))    { evt.preventManipulation();

        inkManager.processPointerUp(evt.currentPoint);

context.lineTo(evt.currentPoint.rawPosition.x, evt.currentPoint.rawPosition.y); context.stroke(); context.closePath();    } else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { // Handle touch up } renderAllStrokes();}

Render everythingfunction renderAllStrokes(){        inkManager.getStrokes().forEach(function (stroke)    {   var first = true;                stroke.getRenderingSegments().forEach(function (segment)        { if (first)            {                context.moveTo(segment.position.x, segment.position.y);                first = false;            }            else            {  context.bezierCurveTo(segment.bezierControlPoint1.x, segment.bezierControlPoint1.y, segment.bezierControlPoint2.x, segment.bezierControlPoint2.y, segment.position.x, segment.position.y);            }        });     }); context.stroke(); context.closePath();}

www.buildwindows.com

Ink Pad app Managing Ink from pen and touch input- Select- Move- Erase

demos

Code Snippets• Select strokes using Touch• Moving selected strokes• Selecting and Erasing modes• Deleting selected strokes

Selecting individual strokes (using touch)function handlePointerDown(evt)

{    . . . else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { var hit = false;    inkManager.getRecognitionResults().forEach(function (result)    { if (inRect(evt.offsetX, evt.offsetY, result.boundingRect))        { result.getStrokes().forEach(function (stroke)            {                 stroke.selected = true; hit = true;            });        }    }); if (hit) { evt.preventManipulation(); prevX = evt.offsetX; prevY = evt.offsetY; } }}

Moving selected strokesfunction handlePointerMove(evt){    if ((evt.pointerType === 3) || ((evt.pointerType === 4) && (evt.button === 1)))    { // Handle pen move    } else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { evt.preventManipulation(); var shift = {x: evt.offsetX - prevX, y: evt.offsetY - prevY};

inkManager.moveSelected(shift);

renderAllStrokes(); prevX = evt.offsetX; prevY = evt.offsetY; }}

Selecting mode (Lasso) and erasing mode

inkManager.mode = Windows.UI.Input.Inking.InkManipulationMode.selecting;

inkManager.mode = Windows.UI.Input.Inking.InkManipulationMode.erasing;

inkManager.mode = Windows.UI.Input.Inking.InkManipulationMode.inking;

// These 3 modes all use the same down/move/up handlers

inkManager.deleteSelected();

www.buildwindows.com

demo

Ink Pad App Converting Ink to text (handwriting recognition) and searching

Code Snippets• Recognition • Find and select all strokes

Recognitionfunction recognize(){        inkManager.recognizeAsync(Windows.UI.Input.Inking.InkRecognitionTarget.all).then(

function (results)    {        inkManager.updateRecognitionResults(results);    });}

Find and select all strokes

function recognizeThenFindAndSelect(target){       inkManager.recognizeAsync(Windows.UI.Input.Inking.InkRecognitionTarget.all).then(function (results)    {    results.forEach(function (result)        {                       result.getTextCandidates().forEach(function (candidate)

            {                if (target.toLowerCase() === candidate.toLowerCase())                {                                       result.getStrokes().forEach(function (stroke)                    {                        stroke.selected = true;                    });                }            });        });    });}

www.buildwindows.com

The new Ink API has been optimized to enable you with easier ramp-up while still covering the most important Metro style app scenarios.

www.buildwindows.com

Call to action

Create Ink enabled Metro style apps • Capture Ink• Manipulate Ink• Recognize Ink

www.buildwindows.com

Related sessions

• [APP-185T] Make great Metro style apps that are touch-optimized using HTML5

• [APP-186T] Build advanced touch apps in Windows 8

• [APP-395T] Designing Metro style: principles and personality

• [APP-740T] Metro style apps using HTML5 from start to finish

www.buildwindows.com

Further reading and documentation

• Quickstart: Capturing and displaying ink • Metro style apps > Learn > Developing basic Metro style apps >

Developing basic apps (JavaScript) > Responding to user input

• Ink App sample in Windows SDK for Metro style apps

• Contact:• For questions, please visit the forums on the Windows Dev Center

at http://forums.dev.windows.com• For best response, please include the Build Session # in the title of

your post

www.buildwindows.com

• Feedback and questions http://forums.dev.windows.com

• Session feedbackhttp://bldw.in/SessionFeedback

thank you

© 2011 Microsoft Corporation. All rights reserved. Microsoft, Windows, Windows Vista and other product names are or may be registered trademarks and/or trademarks in the U.S. and/or other countries.The information herein is for informational purposes only and represents the current view of Microsoft Corporation as of the date of this presentation. Because Microsoft must respond to changing market conditions, it should not be interpreted to

be a commitment on the part of Microsoft, and Microsoft cannot guarantee the accuracy of any information provided after the date of this presentation. MICROSOFT MAKES NO WARRANTIES, EXPRESS, IMPLIED OR STATUTORY, AS TO THE INFORMATION IN THIS PRESENTATION.

www.buildwindows.com

Inking API Architecture – Binary perspective

Metro style app built for Windows using JavaScript / C++ or C# runtime environment

Metro style app

Windows runtime

Windows.UI.Input.Inkingnamespace for ink management

Windows.Graphicsnamespace for rendering

IE events for JavaScript / Windows.Foundationnamespace for ICoreWindow (C++ / C#)

obtains position, id, & pressure from MSPointer (JavaScript)/ ICoreWindow (C++/C#)

manages ink with calls to Windows.UI.Input.Inking

renders using HTML5 Canvas (JavaScript), D2D and DUI

Savefunction saveToFile(){    var picker = new Windows.Storage.Pickers.FileSavePicker();         picker.pickSingleFileAsync().then(function (file)    {             file.openAsync(Windows.Storage.FileAccessMode.readWrite).then(function (stream)        {             var outputStream = fileStream.getOutputStreamAt(0);            

            inkManager.saveAsync(outputStream).then(function()            {                outputStream.flushAsync().then(function()

                {                });            });        });    });}

Loadfunction loadFromFile(){    var picker = new Windows.Storage.Pickers.FileOpenPicker();     picker.pickSingleFileAsync().then(function (file)    {        file.openAsync(Windows.Storage.FileAccessMode.read).then(function (stream)        {            var inputStream = fileStream.getInputStreamAt(0);                        inkManager.Load(inputStream);                         renderAllStrokes();        });    });}

Copy and Paste

function copySelectedToClipboard(){    inkManager.copySelectedToClipboard();}

function pasteFromClipboard(point){    inkManager.pasteFromClipboard(point);}

Select with Recent Recognitionfunction recognizeRecentThenFindAndSelect(target){        inkManager.recognizeAsync(Windows.UI.Input.Inking.InkRecognitionTarget.recent).then(function (results)    {        inkManager.updateRecognitionResults(results);       

                inkManager.getRecognitionResults().forEach(function (result)        {                       result.getTextCandidates().forEach(function (candidate)            {                               if (target.toLowerCase() === candidate. toLowerCase())

                {                    result.getStrokes().forEach(function (stroke)                    {                                               stroke.selected = true; });                }            });        });    });}