This Wiki page is edited by participants of the HTML Accessibility Task Force. It does not necessarily represent consensus and it may have incorrect information or information that is not supported by other Task Force participants, WAI, or W3C. It may also have some very useful information.
ESH Media Multitrack
markup elements
<video> <audio> <cues> <source>
video
Categories Flow content. Phrasing content. Embedded content. If the element has a controls attribute: Interactive content. Contexts in which this element can be used: Where embedded content is expected. Content model: If the element has a src attribute: transparent, but with no media element descendants. If the element does not have a src attribute: zero or more source elements, then transparent, but with no media element descendants.
cues
Contexts in which this element can be used: Where flow content is expected.
Content model: Transparent.
Cues defines an overlay which can be used by a media controller.
source
Simplest possible markup:
<video src=foo.vid >
objects (assumes foo.vid actually has some embedded tracks):
theHTMLMediaElement = { src: theSourceElement, preload: false, mediaGroup='xxx' controller: xxxMediaController, tracks: [xxxAudioTrack1, xxxVideoTrack1], controls: false} theSourceElement = { src: "foo.vid", type:... , media:...} theHTMLVideoElement = { _parent:theHTMLMediaElement, autoplay, loop, width, height, poster, audio } xxxMediaController = { ... , tracks: [xxxAudioTrack1, xxxAudioTrack2, xxxTextTrack1, xxxVideoTrack1, xxxVideoTrack2]} xxxAudioTrack1 = {mediaSource: xxxSrc1, mode=ACTIVE, ...} xxxAudioTrack2 = {mediaSource: xxxSrc2, mode=INACTIVE, ...} xxxTextTrack1 = {mediaSource: xxxSrc3, mode=INACTIVE, ...} xxxVideoTrack1 = {mediaSource: xxxSrc4, mode=ACTIVE, ...} xxxVideoTrack2 = {mediaSource: xxxSrc5, mode=INACTIVE, ...} xxxSrc1 = {netSrc: xxxNetworkObject, src:"foo.vid#audiotrack_1", type:, media:} xxxSrc2 = {netSrc: xxxNetworkObject, src:"foo.vid#audiotrack_2", type:, media:} xxxSrc3 = {netSrc: xxxNetworkObject, src:"foo.vid#texttrack_3", type:, media:} xxxSrc4 = {netSrc: xxxNetworkObject, src:"foo.vid#videotrack_1", type:, media:} xxxSrc5 = {netSrc: xxxNetworkObject, src:"foo.vid#videotrack_2", type:, media:} xxxNetworkObject = {networkState: NETWORK_EMPTY, readyState: HAVE_NOTHING, ... }
gnarly complex case:
<div> <!-- primary content --> <video controls mediagroup="example"> <source src="video.webm" type="video/webm"/> <source src="video.mp4" type="video/mp4"/> </video> <!-- pre-recorded audio descriptions --> <audio kind="descriptions" language="en" label="English Audio Description" checked="" mediagroup="example"> <source src="video.webm#audio_track2" type="audio/ogg"/> <source src="audesc.mp3" type="audio/mp3"/> </audio> <!-- sign language overlay --> <video kind="signing" language="asl" label="American Sign Language" checked="" mediagroup="example"> <source src="signing.webm" type="video/webm"/> <source src="video.mp4#videotrack_2" type="video/mp4"/> </video> <cues kind="captions" language="en" label="Captions" mediagroup="example"> <source src="video.webm#text_track1" type="text/vtt" /> <source src="video.mp4#text_track1" type="application/ttml+xml" /> </cues> </div>
example here of objects (assumes webM/ogg codec is being selected)
me1HTMLMediaElement = { src: me1SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [exampleVideoTrack1, exampleAudioTrack1], controls: true} me1SourceElement1 = { src: "video.webm", type:... , media:...} me1SourceElement2 = { src: "video.mp4", type:... , media:...} me1HTMLVideoElement = { _parent:me1HTMLMediaElement, autoplay, loop, width, height, poster, audio }
me2HTMLMediaElement = { src: me1SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [], controls: true} me2SourceElement1 = { src: "video.webm#audio_track2", type:... , media:...} me2SourceElement2 = { src: "audesc.mp3", type:... , media:...} me2HTMLAudioElement = { _parent:me2HTMLMediaElement, autoplay, loop }
me3HTMLMediaElement = { src: me3SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [], controls: true} me3SourceElement1 = { src: "signing.webm", type:... , media:...} me3SourceElement2 = { src: "video.mp4#videotrack_2", type:... , media:...} me1HTMLVideoElement = { _parent:me3HTMLMediaElement, autoplay, loop, width, height, poster, audio }
me4HTMLMediaElement = { src: me4SourceElement1, preload: false, mediaGroup='example' controller: exampleMediaController, tracks: [], controls: true} me4SourceElement1 = { src: "video.webm#text_track1", type:"text/vtt" , media:"video/webm"} me4SourceElement2 = { src: "video.mp4#text_track1", type:"application/ttml+xml" , media:"video/mp4"} me1HTMLTrackElement = { _parent:me4HTMLMediaElement, ... }
exampleMediaController = { ... , tracks: [exampleVideoTrack1, exampleVideoTrack2, exampleAudioTrack1 ]} exampleVideoTrack1 = {mediaSource: exampleSrc1, mode=ACTIVE, ...} exampleSrc1 = {netSrc: exampleNetworkObject1, src:"video.webm#video_track1", type:, media:} exampleVideoTrack2 = {mediaSource: exampleSrc2, mode=INACTIVE, ...} exampleSrc2 = {netSrc: exampleNetworkObject2, src:"signing.webm#video_track1", type:, media:} exampleAudioTrack1 = {mediaSource: exampleSrc3, mode=ACTIVE, ...} exampleSrc3 = {netSrc: exampleNetworkObject1, src:"video.webm#audio_track1", type:, media:} exampleAudioTrack2 = {mediaSource: exampleSrc4, mode=INACTIVE, ...} exampleSrc4 = {netSrc: exampleNetworkObject1, src:"video.webm#audio_track2", type:, media:} exampleTextTrack1 = {mediaSource: exampleSrc5, mode=INACTIVE, ...} exampleSrc5 = {netSrc: exampleNetworkObject1, src:"video.webm#text_track1", type:, media:}
exampleNetworkObject1 = {networkState: NETWORK_EMPTY, readyState: HAVE_NOTHING, ... } exampleNetworkObject2 = {networkState: NETWORK_EMPTY, readyState: HAVE_NOTHING, ... } // network objects for resources not selected arent instantiated.
HTML Media Element - base class for all the markup objects.
interface HTMLMediaElement : HTMLElement { // attributes available in markup attribute DOMString src; attribute boolean preload; DOMString canPlayType(in DOMString type);
// media controller attribute DOMString mediaGroup; readonly attribute MediaController controller; // assigned depending on mediaGroup. may be null (text track with no media group) readonly attribute MediaTracksCollection tracks; // shorcut to the track(s) which represent this element in the controler // controls attribute boolean controls; // if true this is the element where the controller renders its controls };
Video Element is a object to represent the video element
[NamedConstructor=Video(), NamedConstructor=Video(in DOMString src)] interface HTMLVideoElement : HTMLMediaElement {
attribute boolean autoplay; attribute boolean loop; attribute unsigned long width; attribute unsigned long height; attribute DOMString poster; [PutForwards=value] attribute DOMSettableTokenList audio; };
Audio Element is a object to represent the audio element
[NamedConstructor=Audio(), NamedConstructor=Audio(in DOMString src)] interface HTMLAudioElement : HTMLMediaElement { attribute boolean autoplay; attribute boolean loop; };
Source Element is an object to represent the source element
// can you construct a source element? interface HTMLSourceElement : HTMLMediaElement { attribute DOMString type; attribute DOMString media; };
Track Element is an object to represent the source element
[NamedConstructor=Track(), NamedConstructor=Track(in DOMString src, in DOMString kind, in DOMString lang, in DOMString label)] interface HTMLTrackElement : HTMLMediaElement { attribute DOMString kind; attribute DOMString lang; attribute DOMString label; attribute boolean isdefault; };
Media controller is a shadow DOM element which 'owns' the UA drawn controls
All media elements (if independantly playable) must reference a controller
two media elements can share a controller if they belong to a media group.
This is the only way to control media from script.
interface MediaController { // playback state attribute double currentTime; readonly attribute double initialTime; readonly attribute double duration; readonly attribute Date startOffsetTime; attribute double defaultPlaybackRate; attribute double playbackRate; readonly attribute TimeRanges played; readonly attribute TimeRanges seekable; readonly attribute boolean seeking; readonly attribute boolean ended; attribute boolean autoplay; attribute boolean loop; readonly attribute boolean paused; void play(); void pause(); attribute double volume; attribute boolean muted; void seek(in double delta); attribute Function onplay; attribute Function onpause; attribute Function onratechange; attribute Function onvolumechange; readonly attribute MediaTracksCollection tracks; // All the tracks visible in the media group };
media error records what went wrong in a network source
interface MediaError { const unsigned short MEDIA_ERR_ABORTED = 1; const unsigned short MEDIA_ERR_NETWORK = 2; const unsigned short MEDIA_ERR_DECODE = 3; const unsigned short MEDIA_ERR_SRC_NOT_SUPPORTED = 4; readonly attribute unsigned short code; }
A HTMLNetworkSource is the network connected object that actually gets the bits.
interface HTMLNetworkSource { const unsigned short NETWORK_EMPTY = 0; const unsigned short NETWORK_IDLE = 1; const unsigned short NETWORK_LOADING = 2; const unsigned short NETWORK_NO_SOURCE = 3; readonly attribute unsigned short networkState; attribute DOMString preload; readonly attribute TimeRanges buffered; void load(); // ready state const unsigned short HAVE_NOTHING = 0; const unsigned short HAVE_METADATA = 1; const unsigned short HAVE_CURRENT_DATA = 2; const unsigned short HAVE_FUTURE_DATA = 3; const unsigned short HAVE_ENOUGH_DATA = 4; readonly attribute unsigned short readyState; // error state readonly attribute MediaError error; }
A media source is the url (possibly expanded into a fragment) and type/media info.
interface HTMLMediaSource { readonly attribute HTMLNetworkSource netSrc; attribute DOMString src; // can be a fragment URI attribute DOMString type; attribute DOMString media; }
Base class for all track type objects.
interface HTMLMediaTrack { readonly attribute HTMLMediaSource mediaSource; // may be null ? const unsigned short OFF = 0; const unsigned short INACTIVE = 1; const unsigned short ACTIVE = 2; attribute unsigned short mode; attribute Function onmodechange; };
A text track
interface TextTrack : HTMLMediaTrack { readonly attribute TextTrackCueList cues; readonly attribute TextTrackCueList activeCues; // event raised if a cue becomes active/inactive // with target being the activated/deactivated TextTrackCue attribute Function oncueenter; attribute Function oncueexit; };
A video track
interface VideoTrack : HTMLMediaTrack { // information about the video here. For element see VideoElement readonly attribute unsigned long videoWidth; readonly attribute unsigned long videoHeight; };
An audio track
interface AudioTrack : HTMLMediaTrack { attribute bolean muted; attribute double volume; };
A collection of tracks
interface MediaTracksCollection { readonly attribute unsigned long length; getter HTMLMediaTrack (in unsigned long index); HTMLMediaTrack getTrackById(in DOMString id); };