Hadoop 6 DataNode

2013-07-31

DataNode

public class DataNode extends Configured

implements InterDatanodeProtocol, ClientDatanodeProtocol, FSConstants,
Runnable, DataNodeMXBean

DataNode DataNode
ClientDatanodeProtocol Client InterDatanodeProtocol
DataNode ipcServer
DataNode IPC DataNode
ClientDatanodeProtocol InterDatanodeProtocol
DataNode
DataNode
static{
1

Configuration.addDefaultResource("hdfs-default.xml");
Configuration.addDefaultResource("hdfs-site.xml");
}

DataNode hdfs-default.xml hdfs-site.xml

hdfs-site.xml hdfs-default.xml
hdfs-default.xml src/hdfs main
datanode

1. main
secureMain createDataNode datanode
datanode
2. createDataNode
instantiateDataNode datanode
runDatanodeDaemon datanode
3. instantiateDataNode

${dfs.network.script}
${dfs.data.dir} datanode
makeInstance
4. makeInstance

DataNode DataNode
5.DataNode
startDataNode datanode
shutdown datanode
6.startDataNode
namenode
namenode datanode
machineName:port
namenode version id

DataNode JMX Java Management Extensions Java

datanode ss 50010
DataXceiverServer ss
DataBlockScanner FSDataset
datanode infoServer http://0.0.0.0:50075
2

 https https 50475

infoServer

DataBlockScanner

Servlet
http://0.0.0.0:50075/blockScannerReport
ipc RPC 50020

main secureMain secureMain createDataNode

DataNode createDataNode
instantiateDataNode DataNode runDatanodeDaemon
runDatanodeDaemon NameNode DataNode
DataNode
DataNode instantiateDataNode DataNode
storage makeInstance
makeInstance new
DataNode(conf, dirs);
startDataNode
DataNode DataNode
NameNode socket
NameNode DatanodeProtocol.versionRequest
NamespaceInfo

FSDataset storage data

DataXceiverServer run
DataBlockScanner offerService DataNode
HttpServer ipcServer DataNode
DataNode DataNode NameNode
DataXceiverServer ipcServer
DataNode
DataNode

run

startDistributedUpgradeIfNeeded()/offerService()
3

 offerService offerService
offerService NameNode Block
DataNode Block Block NameNode
DataNode
heartBeatInterval sendHeartbeat
Block receivedBlockList delHints
receivedBlockList DataNode
delHints DataXceiver replaceBlock

datanode.notifyNamenodeReceivedBlock(block, sourceID)

DataNode sourceID BlocksourceID

Block Block DataNode

Block NameNode.blockReceived
Block blockReportInterval
Block NameNode DataNode

DNA_TRANSFER DataNode
DNA_INVALIDATE
DNA_SHUTDOWN DataNode
DNA_REGISTERDataNode
DNA_FINALIZE
DNA_RECOVERBLOCK

DataNode transferBlocks
transferBlocks
Block DataTransfer
DataTransfer DataNode
OP_WRITE_BLOCK
4

 NameNode lease
DataNode
FSDataset: FSDataset
http://caibinbupt.iteye.com/blog/284365
DataXceiverServer:, DataXceiver
http://caibinbupt.iteye.com/blog/284979
DataXceiver:
http://caibinbupt.iteye.com/blog/284979
http://caibinbupt.iteye.com/blog/286533
BlockReceiver:
http://caibinbupt.iteye.com/blog/286259
BlockSender:
DataBlockScanner:
http://caibinbupt.iteye.com/blog/286650