$EVWUDFWLRQV IRU 3URJUDPPLQJ

&RQWHQWV

2YHUYLHZ
$EVWUDFWLRQ /HYHOV
/LEUDULHV
6\VWHP6RIWZDUH
7RRONLWV
+LJKHU 3URJUDPPLQJ /DQJXDJHV
3URJUDPPLQJ /DQJXDJH 5HTXLUHPHQWV
2EMHFWRULHQWHG $SSURDFKHV

 
 2YHUYLHZ

7KH VWDWH RIWKH DUWRISURJUDPPLQJ

– 0RVWRIWKH FXUUHQW FRPPHUFLDOO\ DYDLODEOH PXOWLPHGLD DSSOLFDWLRQV DUH
LPSOHPHQWHG LQSURFHGXUHRULHQWHG SURJUDPPLQJ ODQJXDJHV
– $SSOLFDWLRQ FRGH LV VWLOOKLJKO\ GHSHQGHQW RQKDUGZDUH
– &KDQJHRIPXOWLPHGLD GHYLFHV VWLOORIWHQ UHTXLUHV UHLPSOHPHQWDWLRQ
– &RPPRQRSHUDWLQJ V\VWHP H[WHQVLRQV WU\ WRDWWDFN WKHVH SUREOHPV
– 'LIIHUHQWSURJUDPPLQJ SRVVLELOLWLHV IRU DFFHVVLQJ DQGUHSUHVHQWLQJ PXOWLPHGLD
GDWD
2YHUYLHZ RIGLIIHUHQWDEVWUDFWLRQ OHYHOV
– /LEUDULHV
– 6\VWHP6RIWZDUH
– 7RRONLWV
– +LJKHU 3URJUDPPLQJ /DQJXDJHV
– 2EMHFWRULHQWHG $SSURDFKHV

 
 $EVWUDFWLRQ /HYHOV

0XOWLPHGLD$SSOLFDWLRQ

2EMHFWRULHQWHG +LJKHU
3URJUDPPLQJ 3URJUDPPLQJ
/DQJXDJH /DQJXDJHV

7RRONLWV

6\VWHP6RIWZDUH

/LEUDULHV

'HYLFH 'ULYHUVIRU &RQWLQRXV 0HGLD

'HYLFH

 
 /LEUDULHV

3URFHVVLQJ RIFRQWLQXRXV PHGLDEDVHG RQIXQFWLRQV HPEHGGHG LQWR OLEUDULHV

/LEUDULHV GLIIHU LQWKHLU GHJUHH RIDEVWUDFWLRQ
([DPSOH IURP IBM’s HDUO\ $XGLR9LVXDO&RQQHFWLRQ $9&

acb.channel = AAPI_CHNA
acb.mode = AAPI_PLAY
...
aud_init(&acb) /* acb is the audio control block */
...
audrc = fab_open(AudioFullFileName,AAFB_OPEN,AAFB_EXNO, 0,&fab,0,0,0,0);
fork(START in PARALLEL)
aud_strt(&acb)
displayPosition(RelativeStarttime, Duration)
...

 
 6\VWHP6RIWZDUH

'HYLFH DFFHVV EHFRPHV SDUW RIWKH RSHUDWLQJ V\VWHP

'DWD DV7LPH&DSVXOHV ILOH H[WHQVLRQV
– HDFK /RJLFDO 'DWD 8QLW/'8FDUULHV LQLWV WLPHFDSVXOH LWV GDWD W\SHDFWXDO
YDOXH DQGYDOLG OLIHVSDQ
– XVHIXO FRQFHSW IRU YLGHRZKHUH HDFK IUDPH KDVDYDOLG OLIHVSDQ RIPVUDWHRI
UHDG DFFHVV GXULQJ DQRUPDOSUHVHQWDWLRQ
– SUHVHQWDWLRQ UDWHLV FKDQJHG IRU 9&59LGHR&DVHWWH 5HFRUGHUIXQFWLRQV OLNH
IDVWIRUZDUGVORZ IRUZDUG RU IDVWUHZLQG E\
• FKDQJLQJ WKH SUHVHQWDWLRQ OLIHVSDQ RID/'8
• VNLSSLQJ RI/'8V RU UHSHWLWLRQ RI/'8V

'DWD DV6WUHDPV
– DVWUHDP GHQRWHV WKH FRQWLQXRXV IORZ RIDXGLR DQGYLGHR GDWD EHWZHHQ DVRXUFH
DQGDVLQN
– SULRU WRWKH IORZ WKH VWUHDP LV HVWDEOLVKHG HTXLYDOHQW WRWKH VHWXS RIDFRQQHFWLRQ
LQDQHWZRUNHG HQYLURQPHQW
 
 7RRONLWV

6LPSOHUDSSURDFK WKDQ WKH V\VWHP VRIWZDUH LQWHUIDFH IURP WKH XVHUV SRLQWRIYLHZ
DUH 7RRONLWV VLPSOHUEHFDXVH DEVWDUFWLRQ IURP PDQ\ “XQLQWHUHVWLQJ“ GHWDLOV

– DEVWUDFW IURP WKH DFWXDO SK\VLFDO OD\HU

– DOORZ DXQLIRUPLQWHUIDFH IRU FRPPXQLFDWLRQ ZLWK DOOGLIIHUHQWGHYLFHV RI

FRQWLQXRXV PHGLD

– LQWURGXFH WKH FOLHQWVHUYHU SDUDGLJP

– FDQ KLGH WKH SURFHVVVWUXFWXUHV

– FDQ EH HPEHGGHG LQWR SURJUDPPLQJ ODQJXDJHV RU REMHFWRULHQWHG HQYLURQPHQWV

 
 +LJKHU 3URJUDPPLQJ /DQJXDJHV

0HGLDDV7\SHV
– GHILQLWLRQ RIDSSURSULDWH GDWD W\SHV HJIRU YLGHR DQGDXGLR
– VPDOOHVW XQLW FDQ EH D/'8
– H[DPSOH RIPHUJLQJ DWH[WDQGDPRWLRQ SLFWXUH 2&&$0VLPLODU QRWDWLRQ

subtitle TEXT_STRING 352&(66B

mixed.video, ldu.video VIDEO_LDU; RXWSXWYLGHRBZLQGRZPL[HGYLGHR
... 
WHILE (1'B:+,/(
COBEGIN 
PROCESS_1
input(av_filehandle,ldu.video)
IF new_video_scene
input(subtitle_filehandle,subtitle)
mixed.video := ldu.video + subtitle

,QDERYH H[DPSOH LPSOLFLW W\SH FRQYHUVLRQ PXVW RFFXU

5HDOWLPH SURFHVVLQJ RI/'8V GHSHQG RQWKHLU JUDQXODULW\ DQGFDQ EH
3UREOHPDWLF LQDQ+LJK/HYHO /DQJXDJH +//
 
 +LJKHU 3URJUDPPLQJ /DQJXDJHV

0HGLDDV)LOHV
– LQVWHDG RIFRQVLGHULQJ FRQWLQXRXV PHGLDDVGDWD W\SHV WKH\ FDQ EH FRQVLGHUHG
DVILOHV
file_h1 = open(MICROPHONE_1,...)
file_h2 = open(MICROPHONE_2,...)
file_h3 = open(SPEAKER, ...)
...
read(file_h1)
read(file_h2)
mix(file_h3, file_h1, file_h2)
activate(file_h1, file_h2, file_h3)
...
deactivate(file_h1, file_h2, file_h3)
...
rc1 = close(file_h1)
rc2 = close(file_h2)
rc3 = close(file_h3)

 
 +LJKHU 3URJUDPPLQJ /DQJXDJHV

0HGLDDV3URFHVVHV
– LW LV SRVVLEOH WRPDS FRQWLQRXV PHGLDWRSURFHVVHV DQGLQWHJUDWH WKHP LQWR DQ
+//
7KLV SURFHVV LPSOHPHQWV DVHW RIDFWLRQV
PROCESS cont_process_a;
(“set-volume“, “set-loudness“
...
On_message_do
set_volume ...
set_loudness ...
...
...
[main]
pid = create(cont_process_a)
send(pid, set_volume, 3)
send(pid, set_loudness)
...

 
 3URJUDPPLQJ /DQJXDJH
5HTXLUHPHQWV

7KH +LJK/HYHO/DQJXDJH +//VKRXOG VXSSRUW SDUDOOHOSURFHVVLQJEHFDXVH WKH

SURFHVVLQJ RIFRQWLQXRXV GDWD LV
– FRQWUROOHG E\ WKH +//WKURXJK SXUHDV\QFKURQRXV LQVWUXFWLRQV
– DQLQWHJUDOSDUW RIDSURJUDP WKURXJK WKH LGHQWLILFDWLRQ RIPHGLD
'LIIHUHQWSURFHVVHV PXVW EH DEOH WRFRPPXQLFDWH WKURXJK DQ,QWHU3URFHVV
&RPPXQLFDWLRQ PHFKDQLVP ,3&ZKLFK PXVW EH DEOH WR
– XQGHUVWDQG DSULRULDQGRU LPSOLFLWO\ VSHFLILHG WLPHUHTXLUHPHQWV 4R6
SDUDPHWHUV RU H[WUDFWHG IURP WKH GDWD W\SH
– WUDQVPLW WKH FRQWLQRXV GDWD DFFRUGLQJ WRWKH UHTXLUHPHQWV
– LQLWLDWH WKH SURFHVVLQJ RIWKH UHFHLYHG FRQWLQRXV SURFHVV RQWLPH

 
 2EMHFWRULHQWHG $SSURDFKHV

%DVLFLGHDV RIREMHFWRULHQWHG SURJUDPPLQJ DUH GDWD HQFDSVXODWLRQ LQKHULWDQFHLQ

FRQQHFWLRQ ZLWK FODVV DQGREMHFW GHILQLWLRQV

– $EVWUDFW7\SH'HILQLWLRQGHILQLWLRQ RIGDWD W\SHV WKURXJK DEVWUDFW LQWHUIDFHV

– &ODVV LPSOHPHQWDWLRQ RIDDEVWUDFW GDWD W\SH

– 2EMHFW LQVWDQFH RIDFODVV

2WKHU LPSRUWDQW SURSHUWLHV RIREMHFWRULHQWHG V\VWHPV DUH

– ,QKHULWDQFH

– 3RO\PRUSKLVP

 
 2EMHFWRULHQWHG $SSURDFKHV

'HYLFHV DV&ODVVHV
– GHYLFHV DUH DVVLJQHG WRREMHFWV ZKLFK UHSUHVHQW WKHLU EHKDYLRXU DQGLQWHUIDFH

class media_device {
char *name;
public:
void on(), off();
};

class media_in_device:public media_device { class media_out_device:public media_device{

private: public:
DATA data; void put_data(refDATA dat);
public:
refDATA get_data(); };
};

 
 2EMHFWRULHQWHG $SSURDFKHV

3URFHVVLQJ 8QLWV DV&ODVVHV

7KUHH PDLQ REMHFWV

– VRXUFH REMHFWV
– GHVWLQDWLRQ REMHFWV
– FRPELQHG VRXUFHGHVWLQDWLRQ REMHFWV DOORZV WKH FUHDWLRQ RIGDWD IORZ SDWKV
WKURXJK FRQQHFWLRQ RIREMHFWV

0XOWLPHGLD2EMHFW
– %DVLF0XOWLPHGLD&ODVVHV %0&V
%DVLF0XOWLPHGLD2EMHFWV %02V
– &RPSRXQG0XOWLPHGLD&ODVVHV &0&V
&RPSRXQG0XOWLPHGLD2EMHFWV &02ZKLFK DUH FRPSRXQGRI%0&V %02V
DQGRWKHU &0&V&02V
– %02V DQG&02V FDQ EH GLVWULEXWHG RYHU GLIIHUHQWFRPSXWHU QRGHV

 
 2EMHFWRULHQWHG $SSURDFKHV

([DPSOH RI&0&VLPSOLILHG QRWDWLRQ

Lexicon: compound_class;
DATA: Explain external;
Animation external;

ACCESS_POINTS: VIDEO_SOURCE Animation.VIDEO_SOURCE;

AUDIO_SOURCE Animation.AUDIO_SOURCE;
TEXT_SOURCE Explain.TEXT_SOURCE;
METHODS:
start: display(Explain);
play: play(Animation);
pause: pause(Animation);
stop: stop(Animation);
...

 
 2EMHFWRULHQWHG $SSURDFKHV

0HGLDDV&ODVVHV
– 0HGLD&ODVV +LHUDUFKLHV GHILQH KLHUDUFKLFDO UHODWLRQVIRU GLIIHUHQWPHGLD
– GLIIHUHQWFODVV KLHUDUFKLHV DUH EHWWHU VXLWHG IRU GLIIHUHQWDSSOLFDWLRQV
([DPSOH
Medium Video
Acoustic_Medium Video_Scene
Music
Image
Opus
Note Image_Segment
Audio_Block Pixel
Sample_Value Line
Speech Pixel
...
Column
...
Opitcal_Medium Pixel
Video Animation
Video_Scene ...
Text
...