[Kerberos基础]-- httpclient访问httpfs服务(有Kerberos认证)

场景：cdh集群已经添加kerberos认证，但是需要访问httpfs服务，怎么办？

如下实现：

1、引入maven

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.yjf</groupId>
    <artifactId>test-java</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
        <!--Add  maven release-->
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
        <encoding>UTF-8</encoding>

        <!--Junit -->
        <junit.version>4.12</junit.version>
        <!--Slf4j-->
        <slf4j-version>1.7.20</slf4j-version>
        <!--mysql-->
        <mysql.driver.version>5.1.38</mysql.driver.version>
        <!--httpclient-->
        <httpclient.version>4.3.3</httpclient.version>
        <!--coomon-io -->
        <common.verion>2.5</common.verion>


    </properties>
    <repositories>
        <repository>
            <id>cloudera</id>
            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
        </repository>

        <repository>
            <id>mvnrepository</id>
            <url>https://mvnrepository.com/artifact/</url>
        </repository>

    </repositories>
    <dependencies>
        <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>${common.verion}</version>
        </dependency>

        <!--Httpclient-->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>${httpclient.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpcore</artifactId>
            <version>${httpclient.version}</version>
        </dependency>

        <!--Junit -->
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>${junit.version}</version>
            <scope>test</scope>
        </dependency>
        <!--slf4j-->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>${slf4j-version}</version>
        </dependency>
        <!--mysql-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>${mysql.driver.version}</version>
        </dependency>

    </dependencies>

    <!--maven打包-->
    <build>
        <finalName>test-java</finalName>
        <sourceDirectory>src/main/java</sourceDirectory>
        <testSourceDirectory>src/test/java</testSourceDirectory>
    </build>

</project>

2、实现的代码:

package com.yjf.java.utils;

import java.io.IOException;
import java.io.InputStream;
import java.security.Principal;
import java.security.PrivilegedAction;

import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;


import javax.security.auth.Subject;

import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.Configuration;
import javax.security.auth.login.LoginContext;

import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;

import org.apache.http.client.HttpClient;
import org.apache.http.client.config.AuthSchemes;

import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.config.Lookup;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.impl.auth.SPNegoSchemeFactory;

import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
 * Created by root on 20180118.
 * Update date:
 * <p>
 * Time: 11:21 AM
 * Project: test-java
 * Package: com.yjf.java.utils
 * Describe :
 * <p>
 * Result of Test: test ok
 * Command:
 * <p>
 * <p>
 *
 * Status：Using online
 * Attention：
 */
public class RequestKerberosUrlUtils {
    public static Logger logger = LoggerFactory.getLogger(RequestKerberosUrlUtils.class);
    private String principal;
    private String keyTabLocation;

    public RequestKerberosUrlUtils() {
    }

    public RequestKerberosUrlUtils(String principal, String keyTabLocation) {
        super();
        this.principal = principal;
        this.keyTabLocation = keyTabLocation;
    }

    public RequestKerberosUrlUtils(String principal, String keyTabLocation, boolean isDebug) {
        this(principal, keyTabLocation);
        if (isDebug) {
            System.setProperty("sun.security.spnego.debug", "true");
            System.setProperty("sun.security.krb5.debug", "true");
        }
    }

    public RequestKerberosUrlUtils(String principal, String keyTabLocation, String krb5Location, boolean isDebug) {
        this(principal, keyTabLocation, isDebug);
        System.setProperty("java.security.krb5.conf", krb5Location);
    }

    //模拟curl使用kerberos认证
    private static HttpClient buildSpengoHttpClient() {
        HttpClientBuilder builder = HttpClientBuilder.create();
        Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create().
                register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build();
        builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
        BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() {
            @Override
            public Principal getUserPrincipal() {
                return null;
            }

            @Override
            public String getPassword() {
                return null;
            }
        });
        builder.setDefaultCredentialsProvider(credentialsProvider);
        CloseableHttpClient httpClient = builder.build();
        return httpClient;
    }

    public HttpResponse callRestUrl(final String url, final String userId) {
        logger.warn(String.format("Calling KerberosHttpClient %s %s %s", this.principal, this.keyTabLocation, url));
        Configuration config = new Configuration() {
            @SuppressWarnings("serial")
            @Override
            public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
                return new AppConfigurationEntry[]{new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule",
                        AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, new HashMap<String, Object>() {
                    {
                        put("useTicketCache", "false");
                        put("useKeyTab", "true");
                        put("keyTab", keyTabLocation);
                        //Krb5 in GSS API needs to be refreshed so it does not throw the error
                        //Specified version of key is not available
                        put("refreshKrb5Config", "true");
                        put("principal", principal);
                        put("storeKey", "true");
                        put("doNotPrompt", "true");
                        put("isInitiator", "true");
                        put("debug", "true");
                    }
                })};
            }
        };
        Set<Principal> princ = new HashSet<Principal>(1);
        princ.add(new KerberosPrincipal(userId));
        Subject sub = new Subject(false, princ, new HashSet<Object>(), new HashSet<Object>());
        try {
            //认证模块：Krb5Login
            LoginContext lc = new LoginContext("Krb5Login", sub, null, config);
            lc.login();
            Subject serviceSubject = lc.getSubject();
            return Subject.doAs(serviceSubject, new PrivilegedAction<HttpResponse>() {
                HttpResponse httpResponse = null;

                @Override
                public HttpResponse run() {
                    try {
                        HttpUriRequest request = new HttpGet(url);

                        HttpClient spnegoHttpClient = buildSpengoHttpClient();
                        httpResponse = spnegoHttpClient.execute(request);
                        return httpResponse;
                    } catch (IOException ioe) {
                        ioe.printStackTrace();
                    }
                    return httpResponse;
                }
            });
        } catch (Exception le) {
            le.printStackTrace();
        }
        return null;
    }
}

测试类

package com.test.java.Thread;

import com.yjf.java.utils.RequestKerberosUrlUtils;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


import java.io.InputStream;
import java.util.Arrays;

/**
 * Created by root on 20180118.
 * Update date:
 * <p>
 * Time: 11:34 AM
 * Project: test-java
 * Package: com.test.java.Thread
 * Describe :
 * <p>
 * Result of Test: test ok,test error
 * Command:
 * <p>
 * <p>
 * 
 * Status：Using online
 * Attention：
 */
public class TestSendHttpRequestToUrl {
    private static Logger logger= LoggerFactory.getLogger(TestSendHttpRequestToUrl.class);
    public static void main(String[] args) {
        String user ="[email protected]";
        String keytab="/home/spark/conf/yourUser.keytab";
        String krb5Location="/etc/krb5.conf";

        try{
            RequestKerberosUrlUtils restTest = new RequestKerberosUrlUtils(user,keytab,krb5Location, false);

            // refer to https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Open_and_Read_a_File
            // list dir
            String url_liststatus="http://activeNamenodeHost:50070/webhdfs/v1/user/cdnportal/data/release?op=liststatus";
            // location
            String url_get_block_locations="http://activeNamenodeHost:50070/webhdfs/v1/user/cdnportal/data/release/world.txt?op=get_block_locations";

            HttpResponse response = restTest.callRestUrl(url_liststatus,user);
//          HttpResponse response = restTest.callRestUrl(url_get_block_locations,user);

            InputStream is = response.getEntity().getContent();
            System.out.println("Status code " + response.getStatusLine().getStatusCode());
            System.out.println("message is :"+ Arrays.deepToString(response.getAllHeaders()));
            System.out.println("string：\n"+new String(IOUtils.toByteArray(is), "UTF-8"));

        }catch (Exception exp){
            exp.printStackTrace();
        }

    }
}

[Kerberos基础]-- httpclient访问httpfs服务(有Kerberos认证)

继续阅读

nginx location中斜线的位置的重要性

Apache2.4.x 配置文件详解Apache配置需要了解如下：开始讲解：

配置apache支持PHP（win7）

27 Best Free Eclipse Plug-ins for Java Developer to be ProductiveCode Quality PluginsText Editor PluginsDependency ManagementVersion Control Integration PluginsFramework Development Continuous Integration Related PluginsOther Utility Plugins

Java String.format方法的简单使用

neo4j之cypher使用文档

GitHub连夜封杀！这份阿里 10W 字内部 Java 字面试手册到底有多强？

spark/scala关于【资源文件】加载方法概述外部文件加载方案测试资源文件打包入jar包中小结

mybatis_入门程序Mybatis入门

AOP编程_Android优雅权限框架(1)概念基础，2021金三银四前言正文大纲正文

Effective Java 8:通用程序设计

OOM三种类型

工厂模式-三种类型

【递归】高效率求2的n次幂

win10本地scala和spark安装安装scala安装spark

scala (3) Function 和 Method