From 77491f9028413c50238f2efca9366ab52a536586 Mon Sep 17 00:00:00 2001 From: lhc Date: Mon, 17 May 2021 17:20:05 +0800 Subject: [PATCH] =?UTF-8?q?=E9=80=9A=E7=94=A8=E6=96=B9=E6=B3=95=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=AE=9E=E4=BD=93=E7=B1=BB=E8=AF=BB=E5=8F=96=E6=B3=A8?= =?UTF-8?q?=E8=A7=A3=E5=AD=97=E6=AE=B5=E4=B8=8E=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E4=BA=92=E7=9B=B8=E6=98=A0=E5=B0=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hcframe/base/common/utils/ObjectUtil.java | 8 +- .../java/com/hcframe/spider/SpiderClass.java | 165 +++++++++--------- 2 files changed, 91 insertions(+), 82 deletions(-) diff --git a/hcframe-parent/hcframe-base/src/main/java/com/hcframe/base/common/utils/ObjectUtil.java b/hcframe-parent/hcframe-base/src/main/java/com/hcframe/base/common/utils/ObjectUtil.java index ee65c5f..5168d73 100644 --- a/hcframe-parent/hcframe-base/src/main/java/com/hcframe/base/common/utils/ObjectUtil.java +++ b/hcframe-parent/hcframe-base/src/main/java/com/hcframe/base/common/utils/ObjectUtil.java @@ -5,6 +5,7 @@ import com.hcframe.base.common.ServiceException; import com.hcframe.base.module.data.annotation.DataIgnore; import com.hcframe.base.module.data.exception.BaseMapperException; import com.hcframe.base.module.data.module.DataMap; +import tk.mybatis.mapper.annotation.ColumnType; import javax.persistence.Id; import javax.persistence.Table; @@ -106,7 +107,12 @@ public class ObjectUtil { if (!fieldName.equals(SERIAL_VERSION_UID)) { if (!org.springframework.util.StringUtils.isEmpty(field.get(obj))) { Object value = field.get(obj); - map.put(StringUtils.toUnderScoreUpperCase(fieldName), value); + if (field.getAnnotation(ColumnType.class) != null) { + ColumnType columnType = field.getAnnotation(ColumnType.class); + map.put(columnType.column(), value); + } else { + map.put(StringUtils.toUnderScoreUpperCase(fieldName), value); + } } if (field.getAnnotation(DataIgnore.class) == null) { fieldList.add(StringUtils.toUnderScoreUpperCase(fieldName)); diff --git a/hcframe-parent/hcframe-spider/src/main/java/com/hcframe/spider/SpiderClass.java b/hcframe-parent/hcframe-spider/src/main/java/com/hcframe/spider/SpiderClass.java index afdbeee..1242ac6 100644 --- a/hcframe-parent/hcframe-spider/src/main/java/com/hcframe/spider/SpiderClass.java +++ b/hcframe-parent/hcframe-spider/src/main/java/com/hcframe/spider/SpiderClass.java @@ -15,88 +15,90 @@ import java.util.Set; public class SpiderClass { public static void main(String[] args) { + String str = "000000010001"; + System.out.println(str.length()); // login(); - WebClient client = getInstance(); - try { -// HtmlPage page = client.getPage("https://www.letpub.com.cn/index.php?page=grant#opennewwindow"); - HtmlPage page = client.getPage("https://yswk.csdl.ac.cn/style/module/yswk/page/zylxlist.jsp?id=ZZ"); - Thread.sleep(10000); -// System.out.println("+++++++++++++++++++++++++++++"+page.getTextContent()); - DomElement domElement = page.getElementById("aa"); - System.out.println("+++++++++"+domElement.asText()); - -// Set cookies=null; -// DomNodeList list = page.getElementsByTagName("li"); -// for (DomElement domElement : list) { -// String text = domElement.asText(); -// System.out.println(text); -// } -// HtmlSelect selectA = (HtmlSelect) page.getElementById("addcomment_s1_bysubject_s"); -// selectA.setSelectedAttribute("F", true); -// HtmlSelect selectB = (HtmlSelect) page.getElementById("addcomment_s2_bysubject_s"); -// selectB.setSelectedAttribute("F06", true); -// HtmlOption htmlOption = selectB.getOptionByValue("F06"); -// String selB = htmlOption.asText(); -// HtmlSelect selectC = (HtmlSelect) page.getElementById("addcomment_s3_bysubject_s"); -// List cList = selectC.getOptions(); -// int i = 0; -// for (HtmlOption c : cList) { -// if (i > 0) { -// String selC = c.asText(); -// selectC.setSelectedAttribute(c, true); -// HtmlSelect selectD = (HtmlSelect) page.getElementById("addcomment_s4_bysubject_s"); -// List dList = selectD.getOptions(); -// int j = 0; -// for (HtmlOption d : dList) { -// if (j != 0) { -// String selD = d.asText(); -// selectD.setSelectedAttribute(d, true); -// DomElement element = page.getElementById("searchform_bysubject_s"); -// HtmlInput htmlInput = (HtmlInput) element.querySelector("input[value='advSearch']"); -// htmlInput.click(); -// DomNodeList buttons = page.getElementsByTagName("button"); -// Page p = null; -// for (DomElement domElement : buttons) { -// String str = domElement.asText(); -// if (str.contains("下载Excel")) { -// p = domElement.click(); -// break; -// } -// } -// if (p != null) { -// File file = new File("/Volumes/DATA/letpub/" + selC); -// file.mkdir(); -// saveFile(p,"/Volumes/DATA/letpub/"+selC+"/"+selB+"-"+selC+"-"+selD+".xls"); -// } -// } -// j++; -// Thread.sleep(10*1000); -// } -// } -// i++; -// } -// selectC.setSelectedAttribute("F0101", true); -// HtmlSelect selectD = (HtmlSelect) page.getElementById("addcomment_s4_bysubject_s"); -// selectD.setSelectedAttribute("F010101", true); -// DomElement element = page.getElementById("searchform_bysubject_s"); -// HtmlInput htmlInput = (HtmlInput) element.querySelector("input[value='advSearch']"); -// htmlInput.click(); -// System.out.println("++++++"); -// System.out.println(page.asText()); -// DomNodeList buttons = page.getElementsByTagName("button"); -// Page p = null; -// for (DomElement domElement : buttons) { -// String str = domElement.asText(); -// if (str.contains("下载Excel")) { -// p = domElement.click(); -// break; -// } -// } -// assert p != null; -// saveFile(p,"/Volumes/DATA/a.xls"); - } catch (Exception e) { - e.printStackTrace(); - } +// WebClient client = getInstance(); +// try { +//// HtmlPage page = client.getPage("https://www.letpub.com.cn/index.php?page=grant#opennewwindow"); +// HtmlPage page = client.getPage("https://yswk.csdl.ac.cn/style/module/yswk/page/zylxlist.jsp?id=ZZ"); +// Thread.sleep(10000); +//// System.out.println("+++++++++++++++++++++++++++++"+page.getTextContent()); +// DomElement domElement = page.getElementById("aa"); +// System.out.println("+++++++++"+domElement.asText()); +// +//// Set cookies=null; +//// DomNodeList list = page.getElementsByTagName("li"); +//// for (DomElement domElement : list) { +//// String text = domElement.asText(); +//// System.out.println(text); +//// } +//// HtmlSelect selectA = (HtmlSelect) page.getElementById("addcomment_s1_bysubject_s"); +//// selectA.setSelectedAttribute("F", true); +//// HtmlSelect selectB = (HtmlSelect) page.getElementById("addcomment_s2_bysubject_s"); +//// selectB.setSelectedAttribute("F06", true); +//// HtmlOption htmlOption = selectB.getOptionByValue("F06"); +//// String selB = htmlOption.asText(); +//// HtmlSelect selectC = (HtmlSelect) page.getElementById("addcomment_s3_bysubject_s"); +//// List cList = selectC.getOptions(); +//// int i = 0; +//// for (HtmlOption c : cList) { +//// if (i > 0) { +//// String selC = c.asText(); +//// selectC.setSelectedAttribute(c, true); +//// HtmlSelect selectD = (HtmlSelect) page.getElementById("addcomment_s4_bysubject_s"); +//// List dList = selectD.getOptions(); +//// int j = 0; +//// for (HtmlOption d : dList) { +//// if (j != 0) { +//// String selD = d.asText(); +//// selectD.setSelectedAttribute(d, true); +//// DomElement element = page.getElementById("searchform_bysubject_s"); +//// HtmlInput htmlInput = (HtmlInput) element.querySelector("input[value='advSearch']"); +//// htmlInput.click(); +//// DomNodeList buttons = page.getElementsByTagName("button"); +//// Page p = null; +//// for (DomElement domElement : buttons) { +//// String str = domElement.asText(); +//// if (str.contains("下载Excel")) { +//// p = domElement.click(); +//// break; +//// } +//// } +//// if (p != null) { +//// File file = new File("/Volumes/DATA/letpub/" + selC); +//// file.mkdir(); +//// saveFile(p,"/Volumes/DATA/letpub/"+selC+"/"+selB+"-"+selC+"-"+selD+".xls"); +//// } +//// } +//// j++; +//// Thread.sleep(10*1000); +//// } +//// } +//// i++; +//// } +//// selectC.setSelectedAttribute("F0101", true); +//// HtmlSelect selectD = (HtmlSelect) page.getElementById("addcomment_s4_bysubject_s"); +//// selectD.setSelectedAttribute("F010101", true); +//// DomElement element = page.getElementById("searchform_bysubject_s"); +//// HtmlInput htmlInput = (HtmlInput) element.querySelector("input[value='advSearch']"); +//// htmlInput.click(); +//// System.out.println("++++++"); +//// System.out.println(page.asText()); +//// DomNodeList buttons = page.getElementsByTagName("button"); +//// Page p = null; +//// for (DomElement domElement : buttons) { +//// String str = domElement.asText(); +//// if (str.contains("下载Excel")) { +//// p = domElement.click(); +//// break; +//// } +//// } +//// assert p != null; +//// saveFile(p,"/Volumes/DATA/a.xls"); +// } catch (Exception e) { +// e.printStackTrace(); +// } } private static class innerWebClient{ @@ -163,4 +165,5 @@ public class SpiderClass { output.close(); is.close(); } + }