修复数据平台空指针问题，新增单服务代理模式

hcframe-parent/hcframe-base/pom.xml

@@ -129,11 +129,6 @@
             <artifactId>commons-codec</artifactId>
             <version>1.14</version>
         </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>25.1-jre</version>
-        </dependency>
         <!-- https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc -->
         <dependency>
             <groupId>org.xerial</groupId>
@@ -176,22 +171,6 @@
             <artifactId>ojdbc6</artifactId>
             <version>11.2.0.4</version>
         </dependency>
-        <!-- guava工具类 -->
-        <!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>30.1-jre</version>
-        </dependency>
-
-        <!-- hutool工具类-->
-        <!-- https://mvnrepository.com/artifact/cn.hutool/hutool-all -->
-        <dependency>
-            <groupId>cn.hutool</groupId>
-            <artifactId>hutool-all</artifactId>
-            <version>5.5.9</version>
-        </dependency>
-
     </dependencies>
     <build>
         <finalName>hcframe-base</finalName>

hcframe-parent/hcframe-base/src/main/java/com/hcframe/base/module/data/module/Condition.java

@@ -404,6 +404,9 @@ public class Condition implements Serializable {
         }
 
         public void sqlCheck(Object obj) {
+            if (org.springframework.util.StringUtils.isEmpty(obj)) {
+                obj = "";
+            }
             if (this.flag) {
                 if (XssClass.sqlInj(obj.toString())) {
                     logger.error("非法字符："+obj.toString());
@@ -413,6 +416,9 @@ public class Condition implements Serializable {
         }
 
         public void sqlCheckLike(Object obj) {
+            if (org.springframework.util.StringUtils.isEmpty(obj)) {
+                obj = "";
+            }
             if (this.flag) {
                 if (XssClass.sqlInjLike(obj.toString())) {
                     logger.error("非法字符："+obj.toString());

hcframe-parent/hcframe-config/src/main/java/com/hcframe/config/ConfigApplication.java

@@ -1,6 +1,6 @@
 package com.hcframe.config;
 
-import net.unicon.cas.client.configuration.CasClientConfigurationProperties;
+import net.unicon.cas.client.configuration.EnableCasClient;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
@@ -19,7 +19,7 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2;
 @EnableCaching
 @EnableDiscoveryClient
 @EnableFeignClients
-//@EnableCasClient
+@EnableCasClient
 @ComponentScan(basePackages = {"com.hcframe.**"})
 public class ConfigApplication {
     public static void main(String[] args) {

hcframe-parent/hcframe-config/src/main/java/com/hcframe/config/module/controller/CasController.java

@@ -2,6 +2,7 @@ package com.hcframe.config.module.controller;
 
 import com.hcframe.base.common.ResultVO;
 import com.hcframe.redis.RedisUtil;
+import net.unicon.cas.client.configuration.CasClientConfigurationProperties;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.CookieValue;
@@ -19,6 +20,13 @@ public class CasController {
     @Autowired
     RedisUtil redisUtil;
 
+//    final
+//    CasClientConfigurationProperties casClientConfigurationProperties;
+//
+//    public CasController(CasClientConfigurationProperties casClientConfigurationProperties) {
+//        this.casClientConfigurationProperties = casClientConfigurationProperties;
+//    }
+
     @GetMapping("valid")
     public ResultVO<String> casValid(HttpServletResponse response, HttpServletRequest request) {
         String token = "";

hcframe-parent/hcframe-config/src/main/java/com/hcframe/config/module/controller/TestController.java

@@ -1,14 +1,21 @@
 package com.hcframe.config.module.controller;
 
+import cn.hutool.http.HttpUtil;
 import com.hcframe.base.common.ResultVO;
-import com.hcframe.base.module.auth.entity.FtUser;
 import com.hcframe.base.module.data.module.BaseMapper;
-import org.apache.shiro.SecurityUtils;
+import org.jasig.cas.client.authentication.AttributePrincipal;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RestController;
 
 import javax.servlet.http.HttpServletRequest;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLEncoder;
 
 @RestController
 public class TestController {
@@ -20,4 +27,58 @@ public class TestController {
     public ResultVO<String> getUser(HttpServletRequest request) {
         return ResultVO.getSuccess("token");
     }
+
+    @GetMapping("/casProxy")
+    public ResultVO<Object> getCasProxy(HttpServletRequest request) throws IOException {
+        String serviceUrl = "http://192.168.1.130:8084/user/manage/ja";
+        //1、获取到AttributePrincipal对象
+        AttributePrincipal principal = (AttributePrincipal) request.getUserPrincipal();
+        if (principal == null) {
+            return ResultVO.getFailed("用户未登录");
+        }
+        //2、获取对应的(PT)proxy ticket
+        String proxyTicket = principal.getProxyTicketFor(serviceUrl);
+        if (proxyTicket == null) {
+            return ResultVO.getFailed("PGT 或 PT 不存在");
+        }
+        URL url = new URL(serviceUrl + "?ticket=" + proxyTicket);// 不需要cookie，只需传入代理票据
+
+        HttpURLConnection conn;
+        conn = (HttpURLConnection) url.openConnection();
+        //使用POST方式
+        conn.setRequestMethod("GET");
+        // 设置是否向connection输出，因为这个是post请求，参数要放在
+        // Post 请求不能使用缓存
+        conn.setUseCaches(false);
+
+        //设置本次连接是否自动重定向
+        conn.setInstanceFollowRedirects(true);
+        // 配置本次连接的Content-type，配置为application/x-www-form-urlencoded的
+        // 意思是正文是urlencoded编码过的form参数
+        conn.setRequestProperty("Content-Type","application/x-www-form-urlencoded");
+        // 连接，从postUrl.openConnection()至此的配置必须要在connect之前完成，
+        // 要注意的是connection.getOutputStream会隐含的进行connect。
+        conn.connect();
+
+//        DataOutputStream out = new DataOutputStream(conn
+//                .getOutputStream());
+////         正文，正文内容其实跟get的URL中 '? '后的参数字符串一致
+//        String content = "start=" + URLEncoder.encode("1901-01-01", "UTF-8")+"&end="+URLEncoder.encode("2018-01-01", "UTF-8");
+//        // DataOutputStream.writeBytes将字符串中的16位的unicode字符以8位的字符形式写到流里面
+//        out.writeBytes(content);
+////        流用完记得关
+//        out.flush();
+//        out.close();
+        StringBuilder result = new StringBuilder();
+        //获取响应
+        BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+        String line;
+        while ((line = reader.readLine()) != null){
+            result.append(line);
+        }
+        reader.close();
+        //连接断了
+        conn.disconnect();
+        return ResultVO.getSuccess(result.toString());
+    }
 }

hcframe-parent/hcframe-config/src/main/resources/application.yml

@@ -7,6 +7,9 @@ cas:
   client-host-url: http://192.168.1.130:8081
   validation-type: cas3
   server-login-url: http://192.168.1.131:8080/cas/login
+  # 代理模式
+  proxy-callback-url: http://192.168.1.130:8081/config/proxyCallback
+  proxy-receptor-url: /config/proxyCallback
 pagehelper:
   auto-runtime-dialect: true
 spring:

hcframe-parent/hcframe-gateway/src/main/java/com/hcframe/gateway/GatewayApplication.java

@@ -1,6 +1,6 @@
 package com.hcframe.gateway;
 
-import com.hcframe.gateway_cas_stater.annotate.EnableGatewayCas;
+//import com.hcframe.gateway_cas_stater.annotate.EnableGatewayCas;
 import com.hcframe.redis.RedisUtil;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -10,7 +10,7 @@ import org.springframework.context.annotation.Import;
 @SpringBootApplication
 @EnableDiscoveryClient
 @Import(RedisUtil.class)
-@EnableGatewayCas
+//@EnableGatewayCas
 public class GatewayApplication {
 
     public static void main(String[] args) {

hcframe-parent/hcframe-gateway/src/main/resources/application.yml

@@ -36,6 +36,9 @@ hcframe:
     logout-url: @cas.logout-url@
     cas-service-url: @cas.cas-service-url@
     client-context-path: @cas.client-context-path@
+    accept-any-proxy: true
+    proxyReceptorUrl: /proxyCallback
+    proxyCallbackUrl: http://192.168.1.130:8084/proxyCallback
   config:
     # 是否开启redis 用户登录，若开启此项，需要配置redis节点及相关配置，若不开启，需要注释掉redis相关配置信息
     isRedisLogin: true

hcframe-parent/hcframe-spider/pom.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>parent</artifactId>
+        <groupId>com.hcframe</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>hcframe-spider</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+    <dependencies>
+        <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.13.1</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/net.sourceforge.htmlunit/htmlunit -->
+        <dependency>
+            <groupId>net.sourceforge.htmlunit</groupId>
+            <artifactId>htmlunit</artifactId>
+            <version>2.47.1</version>
+        </dependency>
+
+    </dependencies>
+
+</project>

hcframe-parent/hcframe-spider/src/main/java/com/hcframe/spider/SpiderClass.java

@@ -0,0 +1,168 @@
+package com.hcframe.spider;
+
+import com.gargoylesoftware.htmlunit.BrowserVersion;
+import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
+import com.gargoylesoftware.htmlunit.Page;
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gargoylesoftware.htmlunit.html.*;
+import com.gargoylesoftware.htmlunit.util.Cookie;
+import org.apache.commons.io.IOUtils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Set;
+
+public class SpiderClass {
+
+    public static void main(String[] args)  {
+//        login();
+        WebClient client = getInstance();
+        try {
+//            HtmlPage page  = client.getPage("https://www.letpub.com.cn/index.php?page=grant#opennewwindow");
+            HtmlPage page = client.getPage("https://yswk.csdl.ac.cn/style/module/yswk/page/zylxlist.jsp?id=ZZ");
+            Thread.sleep(10000);
+//            System.out.println("+++++++++++++++++++++++++++++"+page.getTextContent());
+            DomElement domElement = page.getElementById("aa");
+            System.out.println("+++++++++"+domElement.asText());
+
+//            Set<Cookie> cookies=null;
+//            DomNodeList<DomElement> list = page.getElementsByTagName("li");
+//            for (DomElement domElement : list) {
+//                String text = domElement.asText();
+//                System.out.println(text);
+//            }
+//            HtmlSelect selectA = (HtmlSelect) page.getElementById("addcomment_s1_bysubject_s");
+//            selectA.setSelectedAttribute("F", true);
+//            HtmlSelect selectB = (HtmlSelect) page.getElementById("addcomment_s2_bysubject_s");
+//            selectB.setSelectedAttribute("F06", true);
+//            HtmlOption htmlOption = selectB.getOptionByValue("F06");
+//            String selB = htmlOption.asText();
+//            HtmlSelect selectC = (HtmlSelect) page.getElementById("addcomment_s3_bysubject_s");
+//            List<HtmlOption> cList = selectC.getOptions();
+//            int i = 0;
+//            for (HtmlOption c : cList) {
+//                if (i > 0) {
+//                    String selC = c.asText();
+//                    selectC.setSelectedAttribute(c, true);
+//                    HtmlSelect selectD = (HtmlSelect) page.getElementById("addcomment_s4_bysubject_s");
+//                    List<HtmlOption> dList = selectD.getOptions();
+//                    int j = 0;
+//                    for (HtmlOption d : dList) {
+//                        if (j != 0) {
+//                            String selD = d.asText();
+//                            selectD.setSelectedAttribute(d, true);
+//                            DomElement element = page.getElementById("searchform_bysubject_s");
+//                            HtmlInput htmlInput = (HtmlInput) element.querySelector("input[value='advSearch']");
+//                            htmlInput.click();
+//                            DomNodeList<DomElement> buttons = page.getElementsByTagName("button");
+//                            Page p = null;
+//                            for (DomElement domElement : buttons) {
+//                                String str = domElement.asText();
+//                                if (str.contains("下载Excel")) {
+//                                    p = domElement.click();
+//                                    break;
+//                                }
+//                            }
+//                            if (p != null) {
+//                                File file = new File("/Volumes/DATA/letpub/" + selC);
+//                                file.mkdir();
+//                                saveFile(p,"/Volumes/DATA/letpub/"+selC+"/"+selB+"-"+selC+"-"+selD+".xls");
+//                            }
+//                        }
+//                        j++;
+//                        Thread.sleep(10*1000);
+//                    }
+//                }
+//                i++;
+//            }
+//            selectC.setSelectedAttribute("F0101", true);
+//            HtmlSelect selectD = (HtmlSelect) page.getElementById("addcomment_s4_bysubject_s");
+//            selectD.setSelectedAttribute("F010101", true);
+//            DomElement element = page.getElementById("searchform_bysubject_s");
+//            HtmlInput htmlInput = (HtmlInput) element.querySelector("input[value='advSearch']");
+//            htmlInput.click();
+//            System.out.println("++++++");
+//            System.out.println(page.asText());
+//            DomNodeList<DomElement> buttons = page.getElementsByTagName("button");
+//            Page p = null;
+//            for (DomElement domElement : buttons) {
+//                String str = domElement.asText();
+//                if (str.contains("下载Excel")) {
+//                    p = domElement.click();
+//                    break;
+//                }
+//            }
+//            assert p != null;
+//            saveFile(p,"/Volumes/DATA/a.xls");
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private static class innerWebClient{
+        private static final WebClient WEB_CLIENT = new WebClient(BrowserVersion.CHROME);
+    }
+
+
+    public static WebClient getInstance(){
+        return innerWebClient.WEB_CLIENT;
+    }
+
+    /**
+     * 获取指定网页实体
+     * @param url
+     * @return
+     */
+    public static HtmlPage getHtmlPage(String url){
+        return null;
+    }
+
+    public static WebClient getClient() {
+        //调用此方法时加载WebClient
+        WebClient  client= innerWebClient.WEB_CLIENT;
+        client.getCookieManager().setCookiesEnabled(true);
+        // 取消 JS 支持
+        client.getOptions().setRedirectEnabled(true);
+        client.getOptions().setUseInsecureSSL(true);
+        client.getOptions().setCssEnabled(false);
+        // 取消 CSS 支持
+        client.setAjaxController(new NicelyResynchronizingAjaxController());
+        return client;
+    }
+    public static void login()  {
+        WebClient client = getClient();
+        try {
+            HtmlPage page  = client.getPage("https://www.letpub.com.cn/index.php?page=login");
+            Set<Cookie> cookies=null;
+            HtmlInput user = page.getHtmlElementById("email");
+            user.setValueAttribute("taixingyiji@126.com");
+            HtmlInput pwd = page.getHtmlElementById("password");
+            pwd.setValueAttribute("G4VnNjgVLczrNV6");
+            DomNodeList<DomElement> imgs = page.getElementsByTagName("img");
+            HtmlPage htmlPage = null;
+            for (DomElement htmlElement : imgs) {
+                HtmlImage img = (HtmlImage) htmlElement;
+                String src = img.getSrcAttribute();
+                if (src.contains("userlogin")) {
+                    htmlPage = (HtmlPage) img.click();
+                }
+            }
+            System.out.println(htmlPage.asText());
+            cookies = client.getCookieManager().getCookies();
+            for (Cookie cookie : cookies) {
+                System.out.println(cookie.getName()+":"+cookie.getValue());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+    public static void saveFile(Page page, String file) throws Exception {
+        InputStream is = page.getWebResponse().getContentAsStream();
+        FileOutputStream output = new FileOutputStream(file);
+        IOUtils.copy(is, output);
+        output.close();
+        is.close();
+    }
+}

hcframe-parent/hcframe-starter/hcframe-gateway-cas-starter/src/main/java/com/hcframe/gateway_cas_stater/filter/AuthGlobalFilter.java

@@ -89,7 +89,6 @@ public class AuthGlobalFilter implements GlobalFilter, Ordered {
                 token = authId.get(0).getValue();
             }
         }
-
         Assertion assertion = (Assertion) dataStorage.getValue(token, CAS_ASSERTION_KEY);
         if (assertion != null) {
             return chain.filter(exchange);

hcframe-parent/hcframe-starter/hcframe-gateway-cas-starter/src/main/java/com/hcframe/gateway_cas_stater/util/Utils.java

@@ -112,7 +112,8 @@ public class Utils {
         if (!casGatewayClientConfig.getAcceptAnyProxy()) {
             ticketValidator = new Cas30ServiceTicketValidator(casGatewayClientConfig.casServiceUrl + casGatewayClientConfig.casContextPath);
         } else {
-            ticketValidator = new Cas30ProxyTicketValidator(casGatewayClientConfig.casServiceUrl + casGatewayClientConfig.casContextPath);
+            Cas30ProxyTicketValidator cas30ProxyTicketValidator = new Cas30ProxyTicketValidator(casGatewayClientConfig.casServiceUrl + casGatewayClientConfig.casContextPath);
+            ticketValidator = cas30ProxyTicketValidator;
         }
         return ticketValidator;
     }

hcframe-parent/hcframe-user/src/main/java/com/hcframe/user/module/manage/controller/CasController.java

@@ -34,6 +34,7 @@ public class CasController {
     @GetMapping("valid")
     public ResultVO<String> casValid(HttpServletResponse response, HttpServletRequest request,String webUrl) {
         String token = "";
+//        token = request.getHeader("X-Access-Token");
         try {
             Cookie[] cookies = request.getCookies();
             for (Cookie cookie : cookies) {

hcframe-parent/hcframe-user/src/main/resources/application.yml

@@ -5,10 +5,12 @@ server:
 pagehelper:
   auto-runtime-dialect: true
 cas:
-  server-url-prefix: @cas.cas-service-url@
-  client-host-url: @cas.service-url@
+  server-url-prefix: @cas.cas-service-url@@cas.cas-context-path@
+  client-host-url: http://192.168.1.130:8080
   validation-type: cas3
-  server-login-url: @cas.logout-url@
+  server-login-url: @cas.cas-service-url@/cas@cas.login-url@
+  accept-any-proxy: true
+  redirect-after-validation: false
 spring:
   application:
     name: cloud-user

hcframe-parent/pom.xml

@@ -23,6 +23,7 @@
         <module>hcframe-gateway</module>
         <module>hcframe-starter</module>
         <module>hcframe-test</module>
+        <module>hcframe-spider</module>
     </modules>
     <properties>
         <maven.compiler.source>8</maven.compiler.source>
@@ -68,6 +69,21 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-actuator</artifactId>
         </dependency>
+        <!-- guava工具类 -->
+        <!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>30.1-jre</version>
+        </dependency>
+
+        <!-- hutool工具类-->
+        <!-- https://mvnrepository.com/artifact/cn.hutool/hutool-all -->
+        <dependency>
+            <groupId>cn.hutool</groupId>
+            <artifactId>hutool-all</artifactId>
+            <version>5.5.9</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
@@ -144,7 +160,7 @@
                 <cas.cas-service-url>http://192.168.100.98:8080</cas.cas-service-url>
                 <cas.cas-context-path>/cas</cas.cas-context-path>
                 <cas.white-url>^.*(/logout?)$</cas.white-url>
-                <cas.service-url>http://192.168.100.98:8084</cas.service-url>
+                <cas.service-url>http://192.168.1.130:8084</cas.service-url>
                 <cas.cookie-holder-pattern>com.hcframe.gateway.config.MyDataStorage</cas.cookie-holder-pattern>
                 <cas.login-url>/login</cas.login-url>
                 <cas.logout-url>/logout</cas.logout-url>