Hadoop 3RPC

2013-07-25

/Hadoop RPC
IPC Hadoop Writable Hadoop Writable
Java String Writable
IPC Java
Hadoop
RPC CORBA IDL stub
skeleton IOException
RPCorg.apache.hadoop.ipc
Client Server Server RPC Server
RPC
org.apache.hadoop.ipc

 org.apache.hadoop.ipc.Client IPC
org.apache.hadoop.rpc.Client

CallConnectionConnectionId
Call Invocation VO
Connection Thread
ConnectionId
Client Server HDFS NameNode
/ DataNode Client
Client ConnectionId
Connection ID ConnectionId
InetSocketAddressIP ++
InetSocketAddress
Client.Connection RPC Connection
RPC
Connection id
Client.Call Connection
2

 Hash Call
// calls
private Hashtable<Integer, Call> calls = new Hashtable<Integer, Call>();

RPC addCall Connection

Java String Writable
Call ObjectWritable
Client.Connection socket /
Client. writeHeader() Writable
/ Client.Connection
socket Call Call

Call Obejct wait notify RPC

Client Client.Connection

Client cal()
/** Make a call, passing <code>param</code>, to the IPC server defined by
* <code>remoteId</code>, returning the value.
* Throws exceptions if there are network problems or if the remote code
* threw an exception. */
public Writable call(Writable param, ConnectionId remoteId)
throws InterruptedException, IOException {
Call call = new Call(param);
Connection connection = getConnection(remoteId, call);
connection.sendParam(call);
// send the parameter
boolean interrupted = false;
synchronized (call) {
while (!call.done) {
try {
call.wait();
// wait for the result
} catch (InterruptedException ie) {
// save the fact that we were interrupted
interrupted = true;
}
}
3

if (interrupted) {
// set the interrupt flag now that we are done waiting
Thread.currentThread().interrupt();
}
if (call.error != null) {
if (call.error instanceof RemoteException) {
call.error.fillInStackTrace();
throw call.error;
} else { // local exception
// use the connection because it will reflect an ip change, unlike
// the remoteId
throw wrapException(connection.getRemoteAddress(), call.error);
}
} else {
return call.value;
}
}
}

Client.getConnection()

-->

Client.Connection.setupIOstreams()

-->

Client.Connection.setupConnection()

socket
Client.Connection.sendParam() java io socket
Client.Connection.setupIOstreams()

Client.Connection.run()
public void run() {
if (LOG.isDebugEnabled())
LOG.debug(getName() + ": starting, having connections "
+ connections.size());
while (waitForWork()) {//wait here for work - read or close connection
receiveResponse(); //
}
close();

if (LOG.isDebugEnabled())
LOG.debug(getName() + ": stopped, remaining connections "
+ connections.size());
}

Client.Connection. receiveResponse ()
/* Receive a response.
* Because only one receiver, so no synchronization on in.
*/
private void receiveResponse() {
if (shouldCloseConnection.get()) {
return;
}
touch();
try {
int id = in.readInt();

// try to read an id

if (LOG.isDebugEnabled())
LOG.debug(getName() + " got value #" + id);
Call call = calls.get(id);
int state = in.readInt();
// read call status
if (state == Status.SUCCESS.state) {
Writable value = ReflectionUtils.newInstance(valueClass, conf);
value.readFields(in);
// read value
call.setValue(value);
calls.remove(id);
} else if (state == Status.ERROR.state) {
call.setException(new RemoteException(WritableUtils.readString(in),
WritableUtils.readString(in)));
calls.remove(id);
} else if (state == Status.FATAL.state) {
// Close the connection
markClosed(new RemoteException(WritableUtils.readString(in),
WritableUtils.readString(in)));
}
} catch (IOException e) {
markClosed(e);
}
}

call call
5

 client
org.apache.hadoop.ipc. Server

CallListenerResponderConnectionHandler
Call Listener
Listener Listener.Reader
Reader Responder RPC
Responder Connection
Handler callQueue
call
Server
/** Called for each call. */
public abstract Writable call(Class<?> protocol,
Writable param, long receiveTime)
throws IOException;

Server Server
call
6

 Server.Call Client.Call Server.Call

id param Client.Call connection
Call
connection timestamp
response
Writable

Server.Connection socket

Hadoop Server Java NIO socket

socket Server
accept socket
Listener
Server.Handle run
Server.Call Server.call
Responder
NIO
Responder
Server call
call
Server Listener Listener
run()
public void run() {
LOG.info(getName() + ": starting");
SERVER.set(Server.this);
while (running) {
SelectionKey key = null;
try {
selector.select();
Iterator<SelectionKey> iter = selector.selectedKeys().iterator();
7

while (iter.hasNext()) {
key = iter.next();
iter.remove();
try {
if (key.isValid()) {
if (key.isAcceptable())
doAccept(key);
}
} catch (IOException e) {
}
key = null;
}
} catch (OutOfMemoryError e) {
// we can run out of memory if we have too many threads
// log the event and sleep for a minute and give
// some thread(s) a chance to finish
LOG.warn("Out of Memory in server select", e);
closeCurrentConnection(key, e);
cleanupConnections(true);
try { Thread.sleep(60000); } catch (Exception ie) {}
} catch (Exception e) {
closeCurrentConnection(key, e);
}
cleanupConnections(false);
}
LOG.info("Stopping " + this.getName());
synchronized (this) {
try {
acceptChannel.close();
selector.close();
} catch (IOException e) { }
selector= null;
acceptChannel= null;
// clean up all connections
while (!connectionList.isEmpty()) {
closeConnection(connectionList.remove(0));
}
}
}

Listener doAccept ()
8

void doAccept(SelectionKey key) throws IOException, OutOfMemoryError {

Connection c = null;
ServerSocketChannel server = (ServerSocketChannel) key.channel();
SocketChannel channel;
while ((channel = server.accept()) != null) {
channel.configureBlocking(false);
channel.socket().setTcpNoDelay(tcpNoDelay);
Reader reader = getReader();
try {

// readers reader

reader.startAdd(); // readSelector adding true

SelectionKey readKey = reader.registerChannel(channel);//

c = new Connection(readKey, channel, System.currentTimeMillis());//

readKey.attach(c); // connection readKey

synchronized (connectionList) {
connectionList.add(numConnections, c);
numConnections++;
}
if (LOG.isDebugEnabled())
LOG.debug("Server connection from " + c.toString() +
"; # active connections: " + numConnections +
"; # queued calls: " + callQueue.size());
} finally {
// adding false notify() reader, Listener
reader wait()
reader.finishAdd();
}
}
}

reader reader doRead()doRead()

Server.Connection readAndProcess()readAndProcess()
Server.Connection processOneRpc() processData()
processData() call call callQueue
rpc call
Server Handler call
run()

final Call call = callQueue.take(); // pop the queue; maybe blocked here

value = call(call.connection.protocol, call.param,

call.timestamp);

synchronized (call.connection.responseQueue) {
// setupResponse() needs to be sync'ed together with
// responder.doResponse() since setupResponse may use
// SASL to encrypt response data and SASL enforces
// its own message ordering.
setupResponse(buf, call,
(error == null) ? Status.SUCCESS : Status.ERROR,
value, errorClass, error);
// Discard the large buf and reset it back to
// smaller size to freeup heap
if (buf.size() > maxRespSize) {
LOG.warn("Large response size " + buf.size() + " for call " +
call.toString());
buf = new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
}
responder.doRespond(call);
}

Server.Responder doRespond()
void doRespond(Call call) throws IOException {
synchronized (call.connection.responseQueue) {
call.connection.responseQueue.addLast(call);
if (call.connection.responseQueue.size() == 1) {
// writeSelector
processResponse(call.connection.responseQueue, true);
}
}
}
Client Server RPC.java

stub skeleton
CORBA
IDL stub skeleton org.apache.hadoop.rpc
:

10

 InvocationClientCacheInvokerServer
Invocation VO;
ClientCache client socket factory hash key,
hashMap <SocketFactory, Client>; Invoker
InvocationHandler; Server ipc.Server
Hadoop RPC Invoker InvocationHandler invoke invoke
InvocationHandler Invoker
Client
socket proxy Client

Invoker InvocationInvocation
: methodNameparameterClasses
parameters Writable

11

RPC.Server org.apache.hadoop.ipc.Server
RPC Invocation
Java
socket
Dynamic Proxy
Invocation VOClientCache
Server
Invoker Invoker RPC.Invoker
invoke()
public Object invoke(Object proxy, Method method, Object[] args)
throws Throwable {
final boolean logDebug = LOG.isDebugEnabled();
long startTime = 0;
if (logDebug) {
startTime = System.currentTimeMillis();
}
ObjectWritable value = (ObjectWritable)
client.call(new Invocation(method, args), remoteId);
if (logDebug) {
long callTime = System.currentTimeMillis() - startTime;
LOG.debug("Call: " + method.getName() + " " + callTime);
}
return value.get();
}

invoke() method.invoke(ac, arg);

method.invoke(ac,
arg); JVM Hadoop
invoke()

ObjectWritable value = (ObjectWritable)

client.call(new Invocation(method, args), remoteId);

Invocation VO
Client call()
12

RPC.Server org.apache.hadoop.ipc.Server
RPC ipc.RPC getServer()

/** Construct a server for a protocol implementation instance listening on a

* port and address, with a secret manager. */
public static Server getServer(final Object instance, final String bindAddress, final int port,
final int numHandlers,
final boolean verbose, Configuration conf,
SecretManager<? extends TokenIdentifier>
secretManager)
throws IOException {
return new Server(instance, conf, bindAddress, port, numHandlers, verbose,
secretManager);
}

getServer() Server RPC.Server

RPC.Server ipc.Server
RPC. waitForProxy ()
Server Server
RPC.getServer() Server
RPC
RPC IPC
RPC Hadoop

13

VersionedProtocol RPC
getProtocolVersion()
1HDFS
ClientDatanodeProtocol datanode
ClientProtocol client Namenode
DatanodeProtocol : Datanode Namenode
blockreport NamenodeProtocol SecondaryNode
Namenode
(2Mapreduce
InterDatanodeProtocol Datanode block
InnerTrackerProtocol TaskTracker JobTracker
DatanodeProtocol JobSubmissionProtocol JobClient JobTracker
Job Job Job TaskUmbilicalProtocol Task
mapreduce
14

TaskTracker

15