DEMO-Java访问HDFS

适用模块

客户端

具体说明

Java访问HDFS

使用示例

##### 依赖配置（gradle）
```groovy
  implementation 'org.apache.hadoop:hadoop-common:3.0.0'
  implementation 'org.apache.hadoop:hadoop-client:3.0.0'

##### 配置文件
# kerberos配置文件--相对jar包路径
krbFile=/00-conf/krb5.conf
# keytab文件--相对jar包路径
keyFile=/00-conf/hue.keytab
# hdfsUrl
hdfsUrl=hdfs://10.196.83.18:8020

##### 代码示例
package snippet;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.security.UserGroupInformation;
import sun.security.krb5.internal.ktab.KeyTab;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;

/**
 * @Description：通过Java调用api操作hdfs
 * @Author： Semon
 * @Version： v1.0
 * @Date： 2021/9/13 22:14
 */
public class HdfsJavaApi {
    private static String basePath;
    private static String keyFile;
    private static String krbFile;
    private static String hdfsUrl;
    private static Configuration conf;
    private static Path path;
    private static FileSystem fs;

    static {
        try {
            basePath = System.getProperty("user.dir");
            InputStream is = new FileInputStream(basePath + "/00-conf/conf.properties");
            Properties props = new Properties();
            props.load(is);
            keyFile = basePath.concat(props.getProperty("keyFile"));
            krbFile = basePath.concat(props.getProperty("krbFile"));
            hdfsUrl = props.getProperty("hdfsUrl");
            path = new Path(props.getProperty("path"));
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public static void authKrb() throws IOException {
        /**
         * @description：kerberos认证
         * @params: []
         * @return: void
         */
        System.setProperty("java.security.krb5.conf",krbFile);
        conf = new Configuration();
        conf.set("hadoop.security.authentication","kerberos");
        conf.set("fs.defaultFS",hdfsUrl);
        UserGroupInformation.setConfiguration(conf);
        String principal = KeyTab.getInstance(new File(keyFile)).getOneName().getName();
        UserGroupInformation.loginUserFromKeytab(principal,keyFile);
    }

    public static void getHdfsFileList() throws Exception {
        /**
        * @description：获取HDFS文件列表
        * @params: []
        * @return: void
        */
        fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> files = fs.listFiles(path,true);
        while (files.hasNext()) {
            LocatedFileStatus file = files.next();
            System.out.println(file.getPath().toString());
        }
    }

    public static void mkHdfsDir(Path targetDir) throws IOException {
        /**
        * @description：创建hdfs目录
        * @params: [org.apache.hadoop.fs.Path]
        * @return: void
        */
        boolean rs = fs.mkdirs(targetDir);
        System.out.println(rs? "mkdir success!" : "mkdir failed!");
    }

    public static void rmHdfsDir(Path targetDir) throws IOException {
        /**
        * @description：删除hdfs目录
        * @params: [org.apache.hadoop.fs.Path]
        * @return: void
        */
        boolean rs = fs.delete(targetDir,false);
        System.out.println(rs? "delete dir success!" : "delete dir failed!");
    }


    public static void downloadFile(Path remoteDir,Path localDir) throws IOException {
        /**
        * @description： 下载hdfs文件至本地
        * @params: [org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path]
        * @return: void
        */
        fs.copyToLocalFile(remoteDir,localDir);
    }

    public static void uploadFile(Path remotedir,Path localDir) throws IOException {
        /**
        * @description：上传文件至hdfs
        * @params: [org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path]
        * @return: void
        */
        fs.copyFromLocalFile(localDir,remotedir);
    }

    public static void checkFile(Path targetDir) throws IOException {
        /**
        * @description：检查hdfs文件是否存在
        * @params: [org.apache.hadoop.fs.Path]
        * @return: void
        */
        boolean rs = fs.exists(targetDir);
        System.out.println(rs? "File exists!":"File not exists!");
    }


    public static void main(String[] args) throws Exception {
        authKrb();
        getHdfsFileList();
        Path targetPath = new Path(hdfsUrl+"/user/hue/apidemo");
        mkHdfsDir(targetPath);
    }

}

作者：wangsong