Parallelization techniques

Skill 1: threading

no difficult barriers to
cross
f1(…) f2(…)
shared data { {
process
… …
easy parallelism } }
multi-core, multi-CPU,
symmetric, asymmetric
threads = « light proc. »

2
First In First Out scheduling (FIFO)

ready running (1CPU)

thread

thread => priority level

Threads usually not preempted, except:
• by a higher priority thread;
• by an explicit sleep.
T3
T1 T2 TT22
3

Round Robin scheduling, or RR

P1 P2 P3

q > t(P3)
3 context
P1 P2 P3
changes

q = t(P2)
5 context
P1 P2 P3 P1 P3
changes

q < t(P2)
9 context
P1 P2 P3 P1 P2 P3 P1 P3 P3
changes
4
 symmetrical
thread1 … threadN
multi-core/CPU :
OS
OS core1 … coreN

asymmetrical
thread1 … threadN
multi-core/CPU :
OS
master : OS
core1 core2 … coreN
slaves : master

POSIX
pthread_create
pthread_detach

f() f()

pthread_join
leaves the thread
alone
waits for the result
of a thread

6
 Windows
CreateThread
CloseHandle

f() f()

WaitForSingleObject
CloseHandle leaves the thread
alone
waits for the result
of a thread

gcc -S … void *function(void *data)

{
puts("Starting...");
int i;
for (i = 0; i < iterations; ++i)
++count;
puts("End");
return NULL;
.L2:
}
movl count(%rip), %edx
addl $1, %edx
movl %edx, count(%rip)
subl $1,%eax
jne .L2
read increment write

thread 1 edx 1 count edx 2 thread 2

read 1000 1000

increment 1001 1000 1000 read

write 1001 1001 1001 increment

1001 1001 1001 write

8
 prevent allow
interruption interruption

should not be interrupted

read inc write

exclusive release
acquisition of the resource
of a resource
locking unlocking
9

Mutex: mutual exclusion

main

thread1|2

thread2|1
POSIX
void *function(void *data)
{
pthread_mutex_t *mutex = (pthread_mutex_t*)data;

pthread_mutex_lock(mutex);
puts("Starting a print job"); fflush(stdout);
sleep(3);
puts("End of the print job"); fflush(stdout);
pthread_mutex_unlock(mutex);
return NULL;
} 10
int main(void)
{
pthread_t thread1;
pthread_t thread2;
pthread_mutex_t lock;

pthread_mutex_init(& lock, NULL);

pthread_create(&thread1, NULL, function, &lock); started quite

pthread_create(&thread2, NULL, function, &lock);
simultaneously
pthread_join(thread1, NULL);
pthread_join(thread2, NULL);

pthread_mutex_destroy(&lock);

return 0;
}

POSIX

11

thread 1|2 thread 2|1

pthread_mutex_lock(mutex); pthread_mutex_lock(mutex)...
puts("Starting a print job"); ...
sleep(3); ...
puts("End of the print job"); ...
pthread_mutex_unlock(mutex); ...;
puts("Starting a print job");
sleep(3);
puts("End of the print job");
pthread_mutex_unlock(mutex);

12
Mutex : deadlock…

1
mutex 1 4
thread 1

thread 2
3
mutex 2 2

13

Condition variable
pthread_mutex_lock
POSIX pthread_mutex_t pthread_mutex_unlock

mutex

condition
variable

pthread_cond_wait
pthread_cond_timedwait pthread_cond_t
pthread_cond_signal
pthread_cond_broadcast
14
 mutex

POSIX condition
variable

classic creation code sequence :

pthread_mutex_init(mutex, NULL);
pthread_cond_init(conditionVariable, NULL);

classic destruction code sequence :

pthread_cond_destroy(conditionVariable);
pthread_mutex_destroy(mutex);

15

mutex

POSIX condition
variable

classic wait code sequence :

pthread_mutex_lock(mutex);
…
while (condition not true)
pthread_cond_wait(conditionVariable, mutex);
…
pthread_mutex_unlock(mutex);

16
 mutex

POSIX condition
variable

classic signaling code sequence :

pthread_mutex_lock(mutex);
…
if (condition true)
pthread_cond_signal(conditionVariable);
…
pthread_mutex_unlock(mutex);

17

Condition variable
EnterCriticalSection
Vista+ CRITICAL_SECTION LeaveCriticalSection

mutex

condition
variable

SleepConditionVariableCS
WakeConditionVariable CONDITION_VARIABLE
WakeAllConditionVariable
18
 mutex

Vista+ condition
variable

classic creation code sequence :

InitializeCriticalSection(mutex);
InitializeConditionVariable(conditionVariable);

classic destruction code sequence :

// One doesn't have to destroy a condition variable…
DeleteCriticalSection(mutex);

19

mutex

Vista+ condition
variable

classic wait code sequence :

EnterCriticalSection(mutex);
…
while (condition not true)
SleepConditionVariableCS(conditionVariable, mutex, INFINITE);
…
LeaveCriticalSection(mutex);

20
 mutex

Vista+ condition
variable

classic signaling code sequence :

EnterCriticalSection(mutex);
…
if (condition true)
WakeConditionVariable(conditionVariable);
…
LeaveCriticalSection(mutex);

21

Semaphore
semaphore
wait
counter
queue

init value ≥ 0 acces denied to the

semaphore =>
thread (or process)
put in the queue
22
 P V
-- counter ++ counter

counter ≥ 0 ? counter > 0 ?

no no
yes yes
=> enqueue awake an
enqueued
wait unlock element

continue continue
23

classic code sequence

…
P(sem) ask a token

use the
resource
V(sem) free the token
…

24
 counter
≥0 <0
counter = resources still |counter| = waiting
available threads/processes

Initial value :
≥ 1 : resource sharing ;
= 1 : acts like a mutex ;
= 0 : synchronization tool : « consumer thread »
awaits (P), « producer thread » signals (V) :
P consumes P consumes P
produces V produces V …
25