Hadoop 4HDFS

2013-07-29

HDFS NameNode DataNodeNameNode

DataNode
namenode namenode
DataNode
DataNode DataNode
DataNode ID

HDFS DataNode DataNode

NameNode NameNode NameNode NameNode
DataNode
NameNode
HDFS

DataNode NameNode NameNode

NameNode DataNode
DataNodeNameNode
DataNode /
DataNode /
DataNode
DataNode
DataNode DataNode

Hadoop
HADOOP_HOME/conf/hdfs-site.xml
<property>
<name>dfs.data.dir</name>
<value>/usr/local/hadoop/data1,/usr/local/hadoop/data2</value>
</property>

/usr/local/hadoop/data1/usr/local/hadoop/data2
2


drwxr-xr-x 6 hadoop hadoop 4096 4 26 15:11 .
drwxr-xr-x 24 hadoop hadoop 4096 4 19 14:26 ..
drwxrwxr-x 2 hadoop hadoop 4096 4 26 13:57 blocksBeingWritten
drwxrwxr-x 2 hadoop hadoop 4096 4 26 13:57 current
drwxrwxr-x 2 hadoop hadoop 4096 4 3 14:10 detach
-rw-rw-r-- 1 hadoop hadoop 157 4 3 14:10 storage
drwxrwxr-x 2 hadoop hadoop 4096 4 26 13:56 tmp

storage
in_use.lock

blocksBeingWritten block blocksBeingWritten

current current
detach snapshottmp
DataNode tmp
current
subdir0 subdir63
HDFS

HDFS

-rw-rw-r-- 1 hadoop hadoop 66

-rw-rw-r-- 1 hadoop hadoop 11
-rw-rw-r-- 1 hadoop hadoop 727
-rw-rw-r-- 1 hadoop hadoop 15
-rw-rw-r-- 1 hadoop hadoop 7416
-rw-rw-r-- 1 hadoop hadoop 155

4 18 17:19 blk_8027040652559443757
4 18 17:19 blk_8027040652559443757_1055.meta
4 18 17:42 blk_-8559958631634410715
4 18 17:42 blk_-8559958631634410715_1071.meta
6 14 15:36 dncp_block_verification.log.curr
6 14 15:36 VERSION

80270406525594437578559958631634410715
ID 10551071
current
3

VERSION
dncp_block_verification.log.curr DataNode

DataNode HDFS

FORMAT("-format")
REGULAR("-regular")
UPGRADE("-upgrade")
ROLLBACK("-rollback")
FINALIZE("-finalize")
IMPORT("-importCheckpoint") Checkpoint

Hadoop
http://wiki.apache.org/hadoop/Hadoop_Upgradeupgrade
rollback
finalize

importCheckpoint NameNode

//

 Hadoop
DataNode
// DataNode DataStorage

VERSION

NameNode DataNode
DataNode current
previous.tmp snapshot current VERSION

previous.tmp
current previous.tmp
current VERSION
previous.tmp previous

previous current removed.tmp previous

current removed.tmp
previous finalized.tmp

HDFS
previous
5

 DataNode
DataNode
Hadoop Storage
DataNode block
Datanode block
_blk block .meta
meta file blokcFileName_generationStamp.meta
HDFS storage storage
Datanode storage Namenode storage storage

) Datanode FSDatasetFSDataset FSVolume

FSVolume storage FSVolume
FSVolume dataDir blocks
meta file)tmpDirdetachDircopy on write for blocks in snapshot
block detachDir

Storage StorageInfo DataNode

DataStorage Storage

StorageInfo 3
public int layoutVersion; // Version read from the stored file.
public int namespaceID; // namespace id of the storage
public long cTime;
// creation timestamp

Storage dfs.data.dir Storage

StorageDirectory StorageDirectory
analyzeStorage
StorageDirectory

NON_EXISTENT
NOT_FORMATTED
COMPLETE_UPGRADEprevious.tmp current
RECOVER_UPGRADEprevious.tmp current
COMPLETE_FINALIZEfinalized.tmp current
7

COMPLETE_ROLLBACKremoved.tmp current previous

RECOVER_ROLLBACKremoved.tmp current previous
COMPLETE_CHECKPOINTlastcheckpoint.tmp current
RECOVER_CHECKPOINTlastcheckpoint.tmp current
NORMAL

StorageDirectory
current
previous
previous.tmp
removed.tmp
finalized.tmp
lastcheckpoint.tmp NameNode
previous.checkpoint NameNode

doRecover

COMPLETE_UPGRADEmv previous.tmp -> previous

RECOVER_UPGRADEmv previous.tmp -> current
COMPLETE_FINALIZErm finalized.tmp
COMPLETE_ROLLBACKrm removed.tmp
RECOVER_ROLLBACKmv removed.tmp -> current
COMPLETE_CHECKPOINTmv lastcheckpoint.tmp -> previous.checkpoint
RECOVER_CHECKPOINTmv lastcheckpoint.tmp -> current

RECOVER_UPGRADE
1. current->previous.tmp
2. current
3. previous.tmp->previous

previous.tmp current previous.tmp

current
StorageDirectory StorageInfo
StorageDirectory
VERSION StorageDirectory read/write
/ DataNode VERSION
8

#Fri Jun 14 15:36:32 CST 2013

namespaceID=1584403768
storageID=DS-1617068520-127.0.1.1-50010-1364969464023
cTime=0
storageType=DATA_NODE
layoutVersion=-32

StorageDirectory in_use.lock
/StorageDirectory lock
unlock
Storage StorageDirectory Storage

DataStorage Storage DataNode DataNode

// DataStorage doUpgrade/doRollback/doFinalize

DataStorage format DataNode Storage

StorageDirectoryDataStorage
Storage FSDataset
Storage Block
FSDataset

Block Block

blk_3148782637964391313
blk_3148782637964391313_242812.meta
blockId 3148782637964391313242812
numBytesBlock

DatanodeBlockInfo Block Block

FSVolume detach detach
snapshotsnapshot current

current detach
snapshot
current
10

current snapshot detach

copy-on-writeDatanodeBlockInfo
detachBlock Block detach
Block DatanodeBlockInfo FSVolumeSetFSVolume
FSDir DataNode Storage
HDFS Block Storage
FSDataset FSVolume StorageFSDir
FSVolume FSVolumeSet FSDataset FSVolumeSet

FSDir HDFS
FSDir Block
Storage FSDir
FSDir FSDir getBlockInfo
Block getVolumeMap
Block DatanodeBlockInfo

FSVolume Storagedetach
FSVolume
FSVolume
recoverDetachedBlocks detach Storage
detach detach
FSVolume FSVolume
Block FSVolume Block

FSVolumeSet FSVolume
HDFS chunk FSDataset
ActiveFileActiveFile
ActiveFile
11

 FSDatasetFSDataset
FSDatasetInterfaceFSDatasetInterface DataNode
FSDataset
FSVolumeSet volumes;
private HashMap<Block,ActiveFile> ongoingCreates = new HashMap<Block,ActiveFile>();
private HashMap<Block,DatanodeBlockInfo> volumeMap = new HashMap<Block,
DatanodeBlockInfo>();;

volumes FSDataset StorageongoingCreates Block

ActiveFile Block
ongoingCreates
FSDataset
public long getMetaDataLength(Block b) throws IOException;
block block ID
public MetaDataInputStream getMetaDataInputStream(Block b) throws IOException;
block block ID

public boolean metaFileExists(Block b) throws IOException;

block
public long getLength(Block b) throws IOException;
block
public Block getStoredBlock(long blkid) throws IOException;
Block ID Block
public InputStream getBlockInputStream(Block b) throws IOException;
public InputStream getBlockInputStream(Block b, long seekOffset) throws IOException;
Block
public BlockInputStreams getTmpInputStreams(Block b, long blkoff, long ckoff) throws
IOException;
Block tmp
tmp current
current
public BlockWriteStreams writeToBlock(Block b, boolean isRecovery) throws IOException;
12

 block BlockWriteStreams
isRecovery
block
block writeToBlock
ActiveFile
ongoingCreates
BlockWriteStreams
ActiveFile
ActiveFile threads
blk_3148782637964391313 DataNode Block ID
3148782637964391313 DataNode tmp/blk_3148782637964391313
meta tmp/blk_3148782637964391313_XXXXXX.meta
XXXXXX
isRecovery true

finalizeBlock
detached
writeToBlock
interrupt
ongoingCreates
/
ActiveFile ongoingCreates
public void updateBlock(Block oldblock, Block newblock) throws IOException;
block
updateBlock
updateBlock tryUpdateBlock tryUpdateBlock

volumeMap tryUpdateBlock updateBlock

join
public void finalizeBlock(Block b) throws IOException;
finalize writeToBlock block
Block tmp current
FSDataset finalizeBlock ongoingCreates block block
DatanodeBlockInfo volumeMap blk_3148782637964391313
DataNode Block ID 3148782637964391313 DataNode
tmp/blk_3148782637964391313 current subdir12
tmp/blk_3148782637964391313 current/subdir12/blk_3148782637964391313
meta current/subdir12
public void unfinalizeBlock(Block b) throws IOException;
writeToBlock block finalizeBlock
public boolean isValidBlock(Block b);
Block
public void invalidate(Block invalidBlks[]) throws IOException;
13

 block
public void validateBlockMetadata(Block b) throws IOException;
block

DataNode

14