ESH Media Multitrack

markup elements

 <video>
 <audio>
 <cues> 
 <source>

video

Categories
Flow content.
  Phrasing content.
  Embedded content.
  If the element has a controls attribute: Interactive content.
Contexts in which this element can be used:
  Where embedded content is expected.
Content model:
  If the element has a src attribute: transparent, but with no media element descendants.
  If the element does not have a src attribute: zero or more source elements, then transparent, 
  but with no media element descendants.

cues

Contexts in which this element can be used:
Where flow content is expected.

Content model:
Transparent. 

Cues defines an overlay which can be used by a media controller.

source

Simplest possible markup:

<video src=foo.vid >

objects (assumes foo.vid actually has some embedded tracks):

theHTMLMediaElement = { src: theSourceElement, preload: false, mediaGroup='xxx' controller: xxxMediaController, tracks: [xxxAudioTrack1, xxxVideoTrack1], controls: false}
theSourceElement = { src: "foo.vid", type:... , media:...}
theHTMLVideoElement = { _parent:theHTMLMediaElement, autoplay, loop, width, height, poster, audio }
xxxMediaController = { ... , tracks: [xxxAudioTrack1, xxxAudioTrack2, xxxTextTrack1, xxxVideoTrack1, xxxVideoTrack2]}
xxxAudioTrack1 = {mediaSource: xxxSrc1, mode=ACTIVE, ...}
xxxAudioTrack2 = {mediaSource: xxxSrc2, mode=INACTIVE, ...}
xxxTextTrack1 = {mediaSource: xxxSrc3, mode=INACTIVE, ...}
xxxVideoTrack1 = {mediaSource: xxxSrc4, mode=ACTIVE, ...}
xxxVideoTrack2 = {mediaSource: xxxSrc5, mode=INACTIVE, ...} 
xxxSrc1 = {netSrc: xxxNetworkObject, src:"foo.vid#audiotrack_1", type:, media:} 
xxxSrc2 = {netSrc: xxxNetworkObject, src:"foo.vid#audiotrack_2", type:, media:} 
xxxSrc3 = {netSrc: xxxNetworkObject, src:"foo.vid#texttrack_3", type:, media:} 
xxxSrc4 = {netSrc: xxxNetworkObject, src:"foo.vid#videotrack_1", type:, media:} 
xxxSrc5 = {netSrc: xxxNetworkObject, src:"foo.vid#videotrack_2", type:, media:} 
xxxNetworkObject = {networkState: NETWORK_EMPTY, readyState: HAVE_NOTHING, ... }

gnarly complex case:

<div>
   <!-- primary content -->
   <video controls mediagroup="example">
     <source src="video.webm" type="video/webm"/>
     <source src="video.mp4" type="video/mp4"/>
   </video>
   <!-- pre-recorded audio descriptions -->
   <audio kind="descriptions" language="en" label="English Audio Description" checked="" mediagroup="example">
     <source src="video.webm#audio_track2" type="audio/ogg"/>
     <source src="audesc.mp3" type="audio/mp3"/>
   </audio>
   <!-- sign language overlay -->
   <video kind="signing" language="asl" label="American Sign Language" checked="" mediagroup="example">
     <source src="signing.webm" type="video/webm"/>
     <source src="video.mp4#videotrack_2" type="video/mp4"/>
   </video>
   <cues kind="captions" language="en" label="Captions" mediagroup="example">
     <source src="video.webm#text_track1" type="text/vtt" />
     <source src="video.mp4#text_track1" type="application/ttml+xml" />
   </cues>
 </div>

example here of objects (assumes webM/ogg codec is being selected)

me1HTMLMediaElement = { src: me1SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [exampleVideoTrack1, exampleAudioTrack1], controls: true}
me1SourceElement1 = { src: "video.webm", type:... , media:...}
me1SourceElement2 = { src: "video.mp4", type:... , media:...}
me1HTMLVideoElement = { _parent:me1HTMLMediaElement, autoplay, loop, width, height, poster, audio }

me2HTMLMediaElement = { src: me1SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [], controls: true}
me2SourceElement1 = { src: "video.webm#audio_track2", type:... , media:...}
me2SourceElement2 = { src: "audesc.mp3", type:... , media:...}
me2HTMLAudioElement = { _parent:me2HTMLMediaElement, autoplay, loop  }

me3HTMLMediaElement = { src: me3SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [], controls: true}
me3SourceElement1 = { src: "signing.webm", type:... , media:...}
me3SourceElement2 = { src: "video.mp4#videotrack_2", type:... , media:...}
me1HTMLVideoElement = { _parent:me3HTMLMediaElement, autoplay, loop, width, height, poster, audio }

me4HTMLMediaElement = { src: me4SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [], controls: true}
me4SourceElement1 = { src: "video.webm#text_track1", type:"text/vtt" , media:"video/webm"}
me4SourceElement2 = { src: "video.mp4#text_track1", type:"application/ttml+xml" , media:"video/mp4"}
me1HTMLTrackElement = { _parent:me4HTMLMediaElement, ... }

exampleMediaController = { ... , tracks: [exampleVideoTrack1, exampleVideoTrack2, exampleAudioTrack1 ]}
exampleVideoTrack1 = {mediaSource: exampleSrc1, mode=ACTIVE, ...}
exampleSrc1 = {netSrc: exampleNetworkObject1, src:"video.webm#video_track1", type:, media:} 
exampleVideoTrack2 = {mediaSource: exampleSrc2, mode=INACTIVE, ...}
exampleSrc2 = {netSrc: exampleNetworkObject2, src:"signing.webm#video_track1", type:, media:} 
exampleAudioTrack1 = {mediaSource: exampleSrc3, mode=ACTIVE, ...}
exampleSrc3 = {netSrc: exampleNetworkObject1, src:"video.webm#audio_track1", type:, media:} 
exampleAudioTrack2 = {mediaSource: exampleSrc4, mode=INACTIVE, ...}
exampleSrc4 = {netSrc: exampleNetworkObject1, src:"video.webm#audio_track2", type:, media:} 
exampleTextTrack1 = {mediaSource: exampleSrc5, mode=INACTIVE, ...}
exampleSrc5 = {netSrc: exampleNetworkObject1, src:"video.webm#text_track1", type:, media:} 

exampleNetworkObject1 = {networkState: NETWORK_EMPTY, readyState: HAVE_NOTHING, ... }
exampleNetworkObject2 = {networkState: NETWORK_EMPTY, readyState: HAVE_NOTHING, ... }
// network objects for resources not selected arent instantiated.

HTML Media Element - base class for all the markup objects.

interface HTMLMediaElement : HTMLElement {
 // attributes available in markup
          attribute DOMString src;
          attribute boolean preload;
 DOMString canPlayType(in DOMString type);

 // media controller
          attribute DOMString mediaGroup;
 readonly attribute MediaController controller;  // assigned depending on mediaGroup. may be null (text track with no media group)
 readonly attribute MediaTracksCollection tracks;  // shorcut to the track(s) which represent this element in the controler
 // controls
          attribute boolean controls;  // if true this is the element where the controller renders its controls    
 };

Video Element is a object to represent the video element

[NamedConstructor=Video(),
NamedConstructor=Video(in DOMString src)]
interface HTMLVideoElement : HTMLMediaElement {

          attribute boolean autoplay;
          attribute boolean loop;
          attribute unsigned long width;
          attribute unsigned long height;
          attribute DOMString poster;
 [PutForwards=value] attribute DOMSettableTokenList audio; 
 };

Audio Element is a object to represent the audio element

[NamedConstructor=Audio(),
 NamedConstructor=Audio(in DOMString src)]
interface HTMLAudioElement : HTMLMediaElement {
           attribute boolean autoplay;
           attribute boolean loop;

};

Source Element is an object to represent the source element

// can you construct a source element?
interface HTMLSourceElement : HTMLMediaElement {
           attribute DOMString type;
           attribute DOMString media;
};

Track Element is an object to represent the source element

[NamedConstructor=Track(),
 NamedConstructor=Track(in DOMString src, in DOMString kind, in DOMString lang, in DOMString label)]
interface HTMLTrackElement : HTMLMediaElement {
           attribute DOMString kind;
           attribute DOMString lang;
           attribute DOMString label;
           attribute boolean isdefault;
};

Media controller is a shadow DOM element which 'owns' the UA drawn controls All media elements (if independantly playable) must reference a controller two media elements can share a controller if they belong to a media group. This is the only way to control media from script.

interface MediaController {  
// playback state
          attribute double currentTime;
 readonly attribute double initialTime;
 readonly attribute double duration;
 readonly attribute Date startOffsetTime;
          attribute double defaultPlaybackRate;
          attribute double playbackRate;
 readonly attribute TimeRanges played;
 readonly attribute TimeRanges seekable;
 readonly attribute boolean seeking;
 readonly attribute boolean ended;
          attribute boolean autoplay;
          attribute boolean loop;
 readonly attribute boolean paused;
 void play();
 void pause();
          attribute double volume;
          attribute boolean muted;
 void seek(in double delta);
          attribute Function onplay;
          attribute Function onpause;
          attribute Function onratechange;
          attribute Function onvolumechange;
 readonly attribute MediaTracksCollection tracks;  // All the tracks visible in the media group
};

media error records what went wrong in a network source

interface MediaError {
 const unsigned short MEDIA_ERR_ABORTED = 1;
 const unsigned short MEDIA_ERR_NETWORK = 2;
 const unsigned short MEDIA_ERR_DECODE = 3;
 const unsigned short MEDIA_ERR_SRC_NOT_SUPPORTED = 4;
 readonly attribute unsigned short code;
}

A HTMLNetworkSource is the network connected object that actually gets the bits.

interface HTMLNetworkSource {
 const unsigned short NETWORK_EMPTY = 0;
 const unsigned short NETWORK_IDLE = 1;
 const unsigned short NETWORK_LOADING = 2;
 const unsigned short NETWORK_NO_SOURCE = 3; 
 readonly attribute unsigned short networkState;
          attribute DOMString preload;
 readonly attribute TimeRanges buffered;
 void load();
  // ready state
 const unsigned short HAVE_NOTHING = 0;
 const unsigned short HAVE_METADATA = 1;
 const unsigned short HAVE_CURRENT_DATA = 2;
 const unsigned short HAVE_FUTURE_DATA = 3;
 const unsigned short HAVE_ENOUGH_DATA = 4;
 readonly attribute unsigned short readyState;
 // error state
 readonly attribute MediaError error;
}

A media source is the url (possibly expanded into a fragment) and type/media info.

interface HTMLMediaSource {
 readonly attribute HTMLNetworkSource netSrc;
          attribute DOMString src;  // can be a fragment URI
          attribute DOMString type;
          attribute DOMString media;
}

Base class for all track type objects.

interface HTMLMediaTrack {
 readonly attribute HTMLMediaSource mediaSource;  // may be null ?
 const unsigned short OFF = 0;
 const unsigned short INACTIVE = 1;
 const unsigned short ACTIVE = 2;
         attribute unsigned short mode;
         attribute Function onmodechange;
};

A text track

interface TextTrack : HTMLMediaTrack {
 readonly attribute TextTrackCueList cues;
 readonly attribute TextTrackCueList activeCues;
          // event raised if a cue becomes active/inactive
          // with target being the activated/deactivated TextTrackCue
          attribute Function oncueenter;
          attribute Function oncueexit;
};

A video track

interface VideoTrack : HTMLMediaTrack {
   // information about the video here. For element see VideoElement
 readonly attribute unsigned long videoWidth;
 readonly attribute unsigned long videoHeight;
};

An audio track

interface AudioTrack : HTMLMediaTrack {
          attribute bolean muted;
          attribute double volume;
};

A collection of tracks

interface MediaTracksCollection {
 readonly attribute unsigned long length;
 getter HTMLMediaTrack (in unsigned long index);
 HTMLMediaTrack getTrackById(in DOMString id);
};