This Wiki page is edited by participants of the HTML Accessibility Task Force. It does not necessarily represent consensus and it may have incorrect information or information that is not supported by other Task Force participants, WAI, or W3C. It may also have some very useful information.

ESH Media Multitrack

From HTML accessibility task force Wiki
Jump to: navigation, search

markup elements



Flow content.
  Phrasing content.
  Embedded content.
  If the element has a controls attribute: Interactive content.
Contexts in which this element can be used:
  Where embedded content is expected.
Content model:
  If the element has a src attribute: transparent, but with no media element descendants.
  If the element does not have a src attribute: zero or more source elements, then transparent, 
  but with no media element descendants.


Contexts in which this element can be used:
Where flow content is expected.
Content model:

Cues defines an overlay which can be used by a media controller.


Simplest possible markup:

<video src=foo.vid >

objects (assumes foo.vid actually has some embedded tracks):

theHTMLMediaElement = { src: theSourceElement, preload: false, mediaGroup='xxx' controller: xxxMediaController, tracks: [xxxAudioTrack1, xxxVideoTrack1], controls: false}
theSourceElement = { src: "foo.vid", type:... , media:...}
theHTMLVideoElement = { _parent:theHTMLMediaElement, autoplay, loop, width, height, poster, audio }
xxxMediaController = { ... , tracks: [xxxAudioTrack1, xxxAudioTrack2, xxxTextTrack1, xxxVideoTrack1, xxxVideoTrack2]}
xxxAudioTrack1 = {mediaSource: xxxSrc1, mode=ACTIVE, ...}
xxxAudioTrack2 = {mediaSource: xxxSrc2, mode=INACTIVE, ...}
xxxTextTrack1 = {mediaSource: xxxSrc3, mode=INACTIVE, ...}
xxxVideoTrack1 = {mediaSource: xxxSrc4, mode=ACTIVE, ...}
xxxVideoTrack2 = {mediaSource: xxxSrc5, mode=INACTIVE, ...} 
xxxSrc1 = {netSrc: xxxNetworkObject, src:"foo.vid#audiotrack_1", type:, media:} 
xxxSrc2 = {netSrc: xxxNetworkObject, src:"foo.vid#audiotrack_2", type:, media:} 
xxxSrc3 = {netSrc: xxxNetworkObject, src:"foo.vid#texttrack_3", type:, media:} 
xxxSrc4 = {netSrc: xxxNetworkObject, src:"foo.vid#videotrack_1", type:, media:} 
xxxSrc5 = {netSrc: xxxNetworkObject, src:"foo.vid#videotrack_2", type:, media:} 
xxxNetworkObject = {networkState: NETWORK_EMPTY, readyState: HAVE_NOTHING, ... }

gnarly complex case:

   <!-- primary content -->
   <video controls mediagroup="example">
     <source src="video.webm" type="video/webm"/>
     <source src="video.mp4" type="video/mp4"/>
   <!-- pre-recorded audio descriptions -->
   <audio kind="descriptions" language="en" label="English Audio Description" checked="" mediagroup="example">
     <source src="video.webm#audio_track2" type="audio/ogg"/>
     <source src="audesc.mp3" type="audio/mp3"/>
   <!-- sign language overlay -->
   <video kind="signing" language="asl" label="American Sign Language" checked="" mediagroup="example">
     <source src="signing.webm" type="video/webm"/>
     <source src="video.mp4#videotrack_2" type="video/mp4"/>
   <cues kind="captions" language="en" label="Captions" mediagroup="example">
     <source src="video.webm#text_track1" type="text/vtt" />
     <source src="video.mp4#text_track1" type="application/ttml+xml" />

example here of objects (assumes webM/ogg codec is being selected)

me1HTMLMediaElement = { src: me1SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [exampleVideoTrack1, exampleAudioTrack1], controls: true}
me1SourceElement1 = { src: "video.webm", type:... , media:...}
me1SourceElement2 = { src: "video.mp4", type:... , media:...}
me1HTMLVideoElement = { _parent:me1HTMLMediaElement, autoplay, loop, width, height, poster, audio }
me2HTMLMediaElement = { src: me1SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [], controls: true}
me2SourceElement1 = { src: "video.webm#audio_track2", type:... , media:...}
me2SourceElement2 = { src: "audesc.mp3", type:... , media:...}
me2HTMLAudioElement = { _parent:me2HTMLMediaElement, autoplay, loop  }
me3HTMLMediaElement = { src: me3SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [], controls: true}
me3SourceElement1 = { src: "signing.webm", type:... , media:...}
me3SourceElement2 = { src: "video.mp4#videotrack_2", type:... , media:...}
me1HTMLVideoElement = { _parent:me3HTMLMediaElement, autoplay, loop, width, height, poster, audio }
me4HTMLMediaElement = { src: me4SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [], controls: true}
me4SourceElement1 = { src: "video.webm#text_track1", type:"text/vtt" , media:"video/webm"}
me4SourceElement2 = { src: "video.mp4#text_track1", type:"application/ttml+xml" , media:"video/mp4"}
me1HTMLTrackElement = { _parent:me4HTMLMediaElement, ... }
exampleMediaController = { ... , tracks: [exampleVideoTrack1, exampleVideoTrack2, exampleAudioTrack1 ]}
exampleVideoTrack1 = {mediaSource: exampleSrc1, mode=ACTIVE, ...}
exampleSrc1 = {netSrc: exampleNetworkObject1, src:"video.webm#video_track1", type:, media:} 
exampleVideoTrack2 = {mediaSource: exampleSrc2, mode=INACTIVE, ...}
exampleSrc2 = {netSrc: exampleNetworkObject2, src:"signing.webm#video_track1", type:, media:} 
exampleAudioTrack1 = {mediaSource: exampleSrc3, mode=ACTIVE, ...}
exampleSrc3 = {netSrc: exampleNetworkObject1, src:"video.webm#audio_track1", type:, media:} 
exampleAudioTrack2 = {mediaSource: exampleSrc4, mode=INACTIVE, ...}
exampleSrc4 = {netSrc: exampleNetworkObject1, src:"video.webm#audio_track2", type:, media:} 
exampleTextTrack1 = {mediaSource: exampleSrc5, mode=INACTIVE, ...}
exampleSrc5 = {netSrc: exampleNetworkObject1, src:"video.webm#text_track1", type:, media:} 
exampleNetworkObject1 = {networkState: NETWORK_EMPTY, readyState: HAVE_NOTHING, ... }
exampleNetworkObject2 = {networkState: NETWORK_EMPTY, readyState: HAVE_NOTHING, ... }
// network objects for resources not selected arent instantiated.

HTML Media Element - base class for all the markup objects.

interface HTMLMediaElement : HTMLElement {
 // attributes available in markup
          attribute DOMString src;
          attribute boolean preload;
 DOMString canPlayType(in DOMString type);
 // media controller
          attribute DOMString mediaGroup;
 readonly attribute MediaController controller;  // assigned depending on mediaGroup. may be null (text track with no media group)
 readonly attribute MediaTracksCollection tracks;  // shorcut to the track(s) which represent this element in the controler
 // controls
          attribute boolean controls;  // if true this is the element where the controller renders its controls    

Video Element is a object to represent the video element

NamedConstructor=Video(in DOMString src)]
interface HTMLVideoElement : HTMLMediaElement {
          attribute boolean autoplay;
          attribute boolean loop;
          attribute unsigned long width;
          attribute unsigned long height;
          attribute DOMString poster;
 [PutForwards=value] attribute DOMSettableTokenList audio; 

Audio Element is a object to represent the audio element

 NamedConstructor=Audio(in DOMString src)]
interface HTMLAudioElement : HTMLMediaElement {
           attribute boolean autoplay;
           attribute boolean loop;

Source Element is an object to represent the source element
// can you construct a source element?
interface HTMLSourceElement : HTMLMediaElement {
           attribute DOMString type;
           attribute DOMString media;

Track Element is an object to represent the source element

 NamedConstructor=Track(in DOMString src, in DOMString kind, in DOMString lang, in DOMString label)]
interface HTMLTrackElement : HTMLMediaElement {
           attribute DOMString kind;
           attribute DOMString lang;
           attribute DOMString label;
           attribute boolean isdefault;

Media controller is a shadow DOM element which 'owns' the UA drawn controls All media elements (if independantly playable) must reference a controller two media elements can share a controller if they belong to a media group. This is the only way to control media from script.

interface MediaController {  
// playback state
          attribute double currentTime;
 readonly attribute double initialTime;
 readonly attribute double duration;
 readonly attribute Date startOffsetTime;
          attribute double defaultPlaybackRate;
          attribute double playbackRate;
 readonly attribute TimeRanges played;
 readonly attribute TimeRanges seekable;
 readonly attribute boolean seeking;
 readonly attribute boolean ended;
          attribute boolean autoplay;
          attribute boolean loop;
 readonly attribute boolean paused;
 void play();
 void pause();
          attribute double volume;
          attribute boolean muted;
 void seek(in double delta);
          attribute Function onplay;
          attribute Function onpause;
          attribute Function onratechange;
          attribute Function onvolumechange;
 readonly attribute MediaTracksCollection tracks;  // All the tracks visible in the media group

media error records what went wrong in a network source

interface MediaError {
 const unsigned short MEDIA_ERR_ABORTED = 1;
 const unsigned short MEDIA_ERR_NETWORK = 2;
 const unsigned short MEDIA_ERR_DECODE = 3;
 const unsigned short MEDIA_ERR_SRC_NOT_SUPPORTED = 4;
 readonly attribute unsigned short code;

A HTMLNetworkSource is the network connected object that actually gets the bits.

interface HTMLNetworkSource {
 const unsigned short NETWORK_EMPTY = 0;
 const unsigned short NETWORK_IDLE = 1;
 const unsigned short NETWORK_LOADING = 2;
 const unsigned short NETWORK_NO_SOURCE = 3; 
 readonly attribute unsigned short networkState;
          attribute DOMString preload;
 readonly attribute TimeRanges buffered;
 void load();
  // ready state
 const unsigned short HAVE_NOTHING = 0;
 const unsigned short HAVE_METADATA = 1;
 const unsigned short HAVE_CURRENT_DATA = 2;
 const unsigned short HAVE_FUTURE_DATA = 3;
 const unsigned short HAVE_ENOUGH_DATA = 4;
 readonly attribute unsigned short readyState;
 // error state
 readonly attribute MediaError error;

A media source is the url (possibly expanded into a fragment) and type/media info.

interface HTMLMediaSource {
 readonly attribute HTMLNetworkSource netSrc;
          attribute DOMString src;  // can be a fragment URI
          attribute DOMString type;
          attribute DOMString media;

Base class for all track type objects.

interface HTMLMediaTrack {
 readonly attribute HTMLMediaSource mediaSource;  // may be null ?
 const unsigned short OFF = 0;
 const unsigned short INACTIVE = 1;
 const unsigned short ACTIVE = 2;
         attribute unsigned short mode;
         attribute Function onmodechange;

A text track

interface TextTrack : HTMLMediaTrack {
 readonly attribute TextTrackCueList cues;
 readonly attribute TextTrackCueList activeCues;
          // event raised if a cue becomes active/inactive
          // with target being the activated/deactivated TextTrackCue
          attribute Function oncueenter;
          attribute Function oncueexit;

A video track

interface VideoTrack : HTMLMediaTrack {
   // information about the video here. For element see VideoElement
 readonly attribute unsigned long videoWidth;
 readonly attribute unsigned long videoHeight;

An audio track

interface AudioTrack : HTMLMediaTrack {
          attribute bolean muted;
          attribute double volume;

A collection of tracks

interface MediaTracksCollection {
 readonly attribute unsigned long length;
 getter HTMLMediaTrack (in unsigned long index);
 HTMLMediaTrack getTrackById(in DOMString id);