Windows配置本地Hadoop运行环境

2021-03-29 02:26

阅读：667

标签：span ecs apr name override 命令 ram mission res

很多人喜欢用Windows本地开发Hadoop程序，这里是一个在Windows下配置Hadoop的教程。

首先去官网下载hadoop，这里需要下载一个工具winutils，这个工具是编译hadoop用的，下载完之后解压hadoop文件，然后把winutils.exe放到hadoop文件的bin目录下面

然后在hadoop/etc/hadoop下修改以下文件：

core-site.xml:

xml version="1.0" encoding="UTF-8"?>
xml-stylesheet type="text/xsl" href="http://www.soscw.com/configuration.xsl"?>




configuration>
  property>
    name>fs.defaultFSname>
    value>hdfs://localhost:9000/value>
  property>
  property>
    name>io.native.lib.availablename>
    value>falsevalue>
  property>
  property>
    name>hadoop.native.libname>
    value>falsevalue>
  property>
  property>
    name>io.compression.codecsname>
    value>org.apache.hadoop.io.compress.GzipCodec,
           org.apache.hadoop.io.compress.DefaultCodec,
           com.hadoop.compression.lzo.LzoCodec,
           com.hadoop.compression.lzo.LzopCodec,
           org.apache.hadoop.io.compress.BZip2Codec,
           org.apache.hadoop.io.compress.SnappyCodec
        value>
property>
property>
    name>io.compression.codec.lzo.classname>
    value>com.hadoop.compression.lzo.LzoCodecvalue>
property>

configuration>

hdfs-site.xml：

xml version="1.0" encoding="UTF-8"?>
xml-stylesheet type="text/xsl" href="http://www.soscw.com/configuration.xsl"?>




configuration>  
       property>  
               name>dfs.replicationname>  
                value>1value>  
       property>
       property>
                name>dfs.namenode.name.dirname>  
               value>file:///D:/Hadoop/namenodevalue>  
       property>
       property>
               name>dfs.datanode.data.dirname>  
               value>file:///D:/Hadoop/datanodevalue>  
       property>
configuration>

mapred-site.xml：

xml version="1.0"?>
xml-stylesheet type="text/xsl" href="http://www.soscw.com/configuration.xsl"?>




configuration>
    property>
        name>mapreduce.framework.namename>
        value>yarnvalue>
    property>
    property>
        name>mapred.compress.map.outputname>
        value>truevalue>
    property>
    property>
        name>mapred.map.output.compression.codecname>
        value>com.hadoop.compression.lzo.LzoCodecvalue>
    property> 
    property> 
        name>mapred.child.envname> 
        value>LD_LIBRARY_PATH=value>
        name>mapreduce.framework.namename>
        value>yarnvalue>
    property>
    property>
        name>mapred.compress.map.outputname>
        value>truevalue>
    property>
    property>
        name>mapred.map.output.compression.codecname>
        value>com.hadoop.compression.lzo.LzoCodecvalue>
    property> 
    property> 
        name>mapred.child.envname> 
        value>LD_LIBRARY_PATH=D:\hadoop-2.7.3-win64\libvalue> 
    property>
configuration>